github.com/seashell-org/golang-migrate/v4@v4.15.3-0.20220722221203-6ab6c6c062d1/database/spanner/spansql/parser.go (about)

     1  /*
     2  Copyright 2019 Google LLC
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  /*
    18  Package spansql contains types and a parser for the Cloud Spanner SQL dialect.
    19  
    20  To parse, use one of the Parse functions (ParseDDL, ParseDDLStmt, ParseQuery, etc.).
    21  
    22  Sources:
    23  	https://cloud.google.com/spanner/docs/lexical
    24  	https://cloud.google.com/spanner/docs/query-syntax
    25  	https://cloud.google.com/spanner/docs/data-definition-language
    26  */
    27  package spansql
    28  
    29  /*
    30  This file is structured as follows:
    31  
    32  - There are several exported ParseFoo functions that accept an input string
    33    and return a type defined in types.go. This is the principal API of this package.
    34    These functions are implemented as wrappers around the lower-level functions,
    35    with additional checks to ensure things such as input exhaustion.
    36  - The token and parser types are defined. These constitute the lexical token
    37    and parser machinery. parser.next is the main way that other functions get
    38    the next token, with parser.back providing a single token rewind, and
    39    parser.sniff, parser.eat and parser.expect providing lookahead helpers.
    40  - The parseFoo methods are defined, matching the SQL grammar. Each consumes its
    41    namesake production from the parser. There are also some fooParser helper vars
    42    defined that abbreviate the parsing of some of the regular productions.
    43  */
    44  
    45  import (
    46  	"fmt"
    47  	"os"
    48  	"strconv"
    49  	"strings"
    50  	"time"
    51  	"unicode/utf8"
    52  
    53  	"cloud.google.com/go/civil"
    54  )
    55  
    56  const debug = false
    57  
    58  func debugf(format string, args ...interface{}) {
    59  	if !debug {
    60  		return
    61  	}
    62  	fmt.Fprintf(os.Stderr, "spansql debug: "+format+"\n", args...)
    63  }
    64  
    65  // ParseDDL parses a DDL file.
    66  //
    67  // The provided filename is used for error reporting and will
    68  // appear in the returned structure.
    69  func ParseDDLAndDML(filename, s string) (*DDLAndDML, error) {
    70  	p := newParser(filename, s)
    71  
    72  	ddlAndDml := &DDLAndDML{
    73  		Filename: filename,
    74  	}
    75  
    76  	for {
    77  		p.skipSpace()
    78  		if p.done {
    79  			break
    80  		}
    81  
    82  		ddlStmt, err := p.parseDDLStmt()
    83  		if err != nil && err != unknownDDL {
    84  			return nil, err
    85  		}
    86  		if err == nil {
    87  			stmt := DDLOrDMLStmt{
    88  				T:   ddlType,
    89  				Ddl: ddlStmt,
    90  			}
    91  			ddlAndDml.List = append(ddlAndDml.List, stmt)
    92  		} else if err == unknownDDL {
    93  			dmlStmt, err := p.parseDMLStmt()
    94  			if err != nil {
    95  				return nil, err
    96  			}
    97  			stmt := DDLOrDMLStmt{
    98  				T:   dmlType,
    99  				Dml: dmlStmt,
   100  			}
   101  			ddlAndDml.List = append(ddlAndDml.List, stmt)
   102  		}
   103  
   104  		tok := p.next()
   105  		if tok.err == eof {
   106  			break
   107  		} else if tok.err != nil {
   108  			return nil, tok.err
   109  		}
   110  		if tok.value == ";" {
   111  			continue
   112  		} else {
   113  			return nil, p.errorf("unexpected token %q", tok.value)
   114  		}
   115  	}
   116  	if p.Rem() != "" {
   117  		return nil, fmt.Errorf("unexpected trailing contents %q", p.Rem())
   118  	}
   119  
   120  	// Handle comments.
   121  	for _, com := range p.comments {
   122  		c := &Comment{
   123  			Marker:   com.marker,
   124  			Isolated: com.isolated,
   125  			Start:    com.start,
   126  			End:      com.end,
   127  			Text:     com.text,
   128  		}
   129  
   130  		// Strip common whitespace prefix and any whitespace suffix.
   131  		// TODO: This is a bodgy implementation of Longest Common Prefix,
   132  		// and also doesn't do tabs vs. spaces well.
   133  		var prefix string
   134  		for i, line := range c.Text {
   135  			line = strings.TrimRight(line, " \b\t")
   136  			c.Text[i] = line
   137  			trim := len(line) - len(strings.TrimLeft(line, " \b\t"))
   138  			if i == 0 {
   139  				prefix = line[:trim]
   140  			} else {
   141  				// Check how much of prefix is in common.
   142  				for !strings.HasPrefix(line, prefix) {
   143  					prefix = prefix[:len(prefix)-1]
   144  				}
   145  			}
   146  			if prefix == "" {
   147  				break
   148  			}
   149  		}
   150  		if prefix != "" {
   151  			for i, line := range c.Text {
   152  				c.Text[i] = strings.TrimPrefix(line, prefix)
   153  			}
   154  		}
   155  
   156  		ddlAndDml.Comments = append(ddlAndDml.Comments, c)
   157  	}
   158  
   159  	return ddlAndDml, nil
   160  }
   161  
   162  // ParseDDLStmt parses a single DDL statement.
   163  func ParseDDLStmt(s string) (DDLStmt, error) {
   164  	p := newParser("-", s)
   165  	stmt, err := p.parseDDLStmt()
   166  	if err != nil {
   167  		return nil, err
   168  	}
   169  	if p.Rem() != "" {
   170  		return nil, fmt.Errorf("unexpected trailing contents %q", p.Rem())
   171  	}
   172  	return stmt, nil
   173  }
   174  
   175  // ParseDMLStmt parses a single DML statement.
   176  func ParseDMLStmt(s string) (DMLStmt, error) {
   177  	p := newParser("-", s)
   178  	stmt, err := p.parseDMLStmt()
   179  	if err != nil {
   180  		return nil, err
   181  	}
   182  	if p.Rem() != "" {
   183  		return nil, fmt.Errorf("unexpected trailing contents %q", p.Rem())
   184  	}
   185  	return stmt, nil
   186  }
   187  
   188  // ParseQuery parses a query string.
   189  func ParseQuery(s string) (Query, error) {
   190  	p := newParser("-", s)
   191  	q, err := p.parseQuery()
   192  	if err != nil {
   193  		return Query{}, err
   194  	}
   195  	if p.Rem() != "" {
   196  		return Query{}, fmt.Errorf("unexpected trailing query contents %q", p.Rem())
   197  	}
   198  	return q, nil
   199  }
   200  
   201  type token struct {
   202  	value        string
   203  	err          *parseError
   204  	line, offset int
   205  
   206  	typ     tokenType
   207  	float64 float64
   208  	string  string // unquoted form for stringToken/bytesToken/quotedID
   209  
   210  	// int64Token is parsed as a number only when it is known to be a literal.
   211  	// This permits correct handling of operators preceding such a token,
   212  	// which cannot be identified as part of the int64 until later.
   213  	int64Base int
   214  }
   215  
   216  type tokenType int
   217  
   218  const (
   219  	unknownToken tokenType = iota
   220  	int64Token
   221  	float64Token
   222  	stringToken
   223  	bytesToken
   224  	unquotedID
   225  	quotedID
   226  )
   227  
   228  func (t *token) String() string {
   229  	if t.err != nil {
   230  		return fmt.Sprintf("parse error: %v", t.err)
   231  	}
   232  	return strconv.Quote(t.value)
   233  }
   234  
   235  type parseError struct {
   236  	message  string
   237  	filename string
   238  	line     int // 1-based line number
   239  	offset   int // 0-based byte offset from start of input
   240  }
   241  
   242  func (pe *parseError) Error() string {
   243  	if pe == nil {
   244  		return "<nil>"
   245  	}
   246  	if pe.line == 1 {
   247  		return fmt.Sprintf("%s:1.%d: %v", pe.filename, pe.offset, pe.message)
   248  	}
   249  	return fmt.Sprintf("%s:%d: %v", pe.filename, pe.line, pe.message)
   250  }
   251  
   252  var eof = &parseError{message: "EOF"}
   253  var unknownDDL = &parseError{message: "unknown DDL statement"}
   254  
   255  type parser struct {
   256  	s      string // Remaining input.
   257  	done   bool   // Whether the parsing is finished (success or error).
   258  	backed bool   // Whether back() was called.
   259  	cur    token
   260  
   261  	filename     string
   262  	line, offset int // updated by places that shrink s
   263  
   264  	comments []comment // accumulated during parse
   265  }
   266  
   267  type comment struct {
   268  	marker     string // "#" or "--" or "/*"
   269  	isolated   bool   // if it starts on its own line
   270  	start, end Position
   271  	text       []string
   272  }
   273  
   274  // Pos reports the position of the current token.
   275  func (p *parser) Pos() Position { return Position{Line: p.cur.line, Offset: p.cur.offset} }
   276  
   277  func newParser(filename, s string) *parser {
   278  	return &parser{
   279  		s: s,
   280  
   281  		cur: token{line: 1},
   282  
   283  		filename: filename,
   284  		line:     1,
   285  	}
   286  }
   287  
   288  // Rem returns the unparsed remainder, ignoring space.
   289  func (p *parser) Rem() string {
   290  	rem := p.s
   291  	if p.backed {
   292  		rem = p.cur.value + rem
   293  	}
   294  	i := 0
   295  	for ; i < len(rem); i++ {
   296  		if !isSpace(rem[i]) {
   297  			break
   298  		}
   299  	}
   300  	return rem[i:]
   301  }
   302  
   303  func (p *parser) String() string {
   304  	if p.backed {
   305  		return fmt.Sprintf("next tok: %s (rem: %q)", &p.cur, p.s)
   306  	}
   307  	return fmt.Sprintf("rem: %q", p.s)
   308  }
   309  
   310  func (p *parser) errorf(format string, args ...interface{}) *parseError {
   311  	pe := &parseError{
   312  		message:  fmt.Sprintf(format, args...),
   313  		filename: p.filename,
   314  		line:     p.cur.line,
   315  		offset:   p.cur.offset,
   316  	}
   317  	p.cur.err = pe
   318  	p.done = true
   319  	return pe
   320  }
   321  
   322  func isInitialIdentifierChar(c byte) bool {
   323  	// https://cloud.google.com/spanner/docs/lexical#identifiers
   324  	switch {
   325  	case 'A' <= c && c <= 'Z':
   326  		return true
   327  	case 'a' <= c && c <= 'z':
   328  		return true
   329  	case c == '_':
   330  		return true
   331  	}
   332  	return false
   333  }
   334  
   335  func isIdentifierChar(c byte) bool {
   336  	// https://cloud.google.com/spanner/docs/lexical#identifiers
   337  	// This doesn't apply the restriction that an identifier cannot start with [0-9],
   338  	// nor does it check against reserved keywords.
   339  	switch {
   340  	case 'A' <= c && c <= 'Z':
   341  		return true
   342  	case 'a' <= c && c <= 'z':
   343  		return true
   344  	case '0' <= c && c <= '9':
   345  		return true
   346  	case c == '_':
   347  		return true
   348  	}
   349  	return false
   350  }
   351  
   352  func isHexDigit(c byte) bool {
   353  	return '0' <= c && c <= '9' || 'a' <= c && c <= 'f' || 'A' <= c && c <= 'F'
   354  }
   355  
   356  func isOctalDigit(c byte) bool {
   357  	return '0' <= c && c <= '7'
   358  }
   359  
   360  func (p *parser) consumeNumber() {
   361  	/*
   362  		int64_value:
   363  			{ decimal_value | hex_value }
   364  
   365  		decimal_value:
   366  			[-]0—9+
   367  
   368  		hex_value:
   369  			[-]0[xX]{0—9|a—f|A—F}+
   370  
   371  		(float64_value is not formally specified)
   372  
   373  		float64_value :=
   374  			  [+-]DIGITS.[DIGITS][e[+-]DIGITS]
   375  			| [DIGITS].DIGITS[e[+-]DIGITS]
   376  			| DIGITSe[+-]DIGITS
   377  	*/
   378  
   379  	i, neg, base := 0, false, 10
   380  	float, e, dot := false, false, false
   381  	if p.s[i] == '-' {
   382  		neg = true
   383  		i++
   384  	} else if p.s[i] == '+' {
   385  		// This isn't in the formal grammar, but is mentioned informally.
   386  		// https://cloud.google.com/spanner/docs/lexical#integer-literals
   387  		i++
   388  	}
   389  	if strings.HasPrefix(p.s[i:], "0x") || strings.HasPrefix(p.s[i:], "0X") {
   390  		base = 16
   391  		i += 2
   392  	}
   393  	d0 := i
   394  digitLoop:
   395  	for i < len(p.s) {
   396  		switch c := p.s[i]; {
   397  		case '0' <= c && c <= '9':
   398  			i++
   399  		case base == 16 && 'A' <= c && c <= 'F':
   400  			i++
   401  		case base == 16 && 'a' <= c && c <= 'f':
   402  			i++
   403  		case base == 10 && (c == 'e' || c == 'E'):
   404  			if e {
   405  				p.errorf("bad token %q", p.s[:i])
   406  				return
   407  			}
   408  			// Switch to consuming float.
   409  			float, e = true, true
   410  			i++
   411  
   412  			if i < len(p.s) && (p.s[i] == '+' || p.s[i] == '-') {
   413  				i++
   414  			}
   415  		case base == 10 && c == '.':
   416  			if dot || e { // any dot must come before E
   417  				p.errorf("bad token %q", p.s[:i])
   418  				return
   419  			}
   420  			// Switch to consuming float.
   421  			float, dot = true, true
   422  			i++
   423  		default:
   424  			break digitLoop
   425  		}
   426  	}
   427  	if d0 == i {
   428  		p.errorf("no digits in numeric literal")
   429  		return
   430  	}
   431  	sign := ""
   432  	if neg {
   433  		sign = "-"
   434  	}
   435  	p.cur.value, p.s = p.s[:i], p.s[i:]
   436  	p.offset += i
   437  	var err error
   438  	if float {
   439  		p.cur.typ = float64Token
   440  		p.cur.float64, err = strconv.ParseFloat(sign+p.cur.value[d0:], 64)
   441  	} else {
   442  		p.cur.typ = int64Token
   443  		p.cur.value = sign + p.cur.value[d0:]
   444  		p.cur.int64Base = base
   445  		// This is parsed on demand.
   446  	}
   447  	if err != nil {
   448  		p.errorf("bad numeric literal %q: %v", p.cur.value, err)
   449  	}
   450  }
   451  
   452  func (p *parser) consumeString() {
   453  	// https://cloud.google.com/spanner/docs/lexical#string-and-bytes-literals
   454  
   455  	delim := p.stringDelimiter()
   456  	if p.cur.err != nil {
   457  		return
   458  	}
   459  
   460  	p.cur.string, p.cur.err = p.consumeStringContent(delim, false, true, "string literal")
   461  	p.cur.typ = stringToken
   462  }
   463  
   464  func (p *parser) consumeRawString() {
   465  	// https://cloud.google.com/spanner/docs/lexical#string-and-bytes-literals
   466  
   467  	p.s = p.s[1:] // consume 'R'
   468  	delim := p.stringDelimiter()
   469  	if p.cur.err != nil {
   470  		return
   471  	}
   472  
   473  	p.cur.string, p.cur.err = p.consumeStringContent(delim, true, true, "raw string literal")
   474  	p.cur.typ = stringToken
   475  }
   476  
   477  func (p *parser) consumeBytes() {
   478  	// https://cloud.google.com/spanner/docs/lexical#string-and-bytes-literals
   479  
   480  	p.s = p.s[1:] // consume 'B'
   481  	delim := p.stringDelimiter()
   482  	if p.cur.err != nil {
   483  		return
   484  	}
   485  
   486  	p.cur.string, p.cur.err = p.consumeStringContent(delim, false, false, "bytes literal")
   487  	p.cur.typ = bytesToken
   488  }
   489  
   490  func (p *parser) consumeRawBytes() {
   491  	// https://cloud.google.com/spanner/docs/lexical#string-and-bytes-literals
   492  
   493  	p.s = p.s[2:] // consume 'RB'
   494  	delim := p.stringDelimiter()
   495  	if p.cur.err != nil {
   496  		return
   497  	}
   498  
   499  	p.cur.string, p.cur.err = p.consumeStringContent(delim, true, false, "raw bytes literal")
   500  	p.cur.typ = bytesToken
   501  }
   502  
   503  // stringDelimiter returns the opening string delimiter.
   504  func (p *parser) stringDelimiter() string {
   505  	c := p.s[0]
   506  	if c != '"' && c != '\'' {
   507  		p.errorf("invalid string literal")
   508  		return ""
   509  	}
   510  	// Look for triple.
   511  	if len(p.s) >= 3 && p.s[1] == c && p.s[2] == c {
   512  		return p.s[:3]
   513  	}
   514  	return p.s[:1]
   515  }
   516  
   517  // consumeStringContent consumes a string-like literal, including its delimiters.
   518  //
   519  //   - delim is the opening/closing delimiter.
   520  //   - raw is true if consuming a raw string.
   521  //   - unicode is true if unicode escape sequence (\uXXXX or \UXXXXXXXX) are permitted.
   522  //   - name identifies the name of the consuming token.
   523  //
   524  // It is designed for consuming string, bytes literals, and also backquoted identifiers.
   525  func (p *parser) consumeStringContent(delim string, raw, unicode bool, name string) (string, *parseError) {
   526  	// https://cloud.google.com/spanner/docs/lexical#string-and-bytes-literals
   527  
   528  	if len(delim) == 3 {
   529  		name = "triple-quoted " + name
   530  	}
   531  
   532  	i := len(delim)
   533  	var content []byte
   534  
   535  	for i < len(p.s) {
   536  		if strings.HasPrefix(p.s[i:], delim) {
   537  			i += len(delim)
   538  			p.s = p.s[i:]
   539  			p.offset += i
   540  			return string(content), nil
   541  		}
   542  
   543  		if p.s[i] == '\\' {
   544  			i++
   545  			if i >= len(p.s) {
   546  				return "", p.errorf("unclosed %s", name)
   547  			}
   548  
   549  			if raw {
   550  				content = append(content, '\\', p.s[i])
   551  				i++
   552  				continue
   553  			}
   554  
   555  			switch p.s[i] {
   556  			case 'a':
   557  				i++
   558  				content = append(content, '\a')
   559  			case 'b':
   560  				i++
   561  				content = append(content, '\b')
   562  			case 'f':
   563  				i++
   564  				content = append(content, '\f')
   565  			case 'n':
   566  				i++
   567  				content = append(content, '\n')
   568  			case 'r':
   569  				i++
   570  				content = append(content, '\r')
   571  			case 't':
   572  				i++
   573  				content = append(content, '\t')
   574  			case 'v':
   575  				i++
   576  				content = append(content, '\v')
   577  			case '\\':
   578  				i++
   579  				content = append(content, '\\')
   580  			case '?':
   581  				i++
   582  				content = append(content, '?')
   583  			case '"':
   584  				i++
   585  				content = append(content, '"')
   586  			case '\'':
   587  				i++
   588  				content = append(content, '\'')
   589  			case '`':
   590  				i++
   591  				content = append(content, '`')
   592  			case 'x', 'X':
   593  				i++
   594  				if !(i+1 < len(p.s) && isHexDigit(p.s[i]) && isHexDigit(p.s[i+1])) {
   595  					return "", p.errorf("illegal escape sequence: hex escape sequence must be followed by 2 hex digits")
   596  				}
   597  				c, err := strconv.ParseUint(p.s[i:i+2], 16, 64)
   598  				if err != nil {
   599  					return "", p.errorf("illegal escape sequence: invalid hex digits: %q: %v", p.s[i:i+2], err)
   600  				}
   601  				content = append(content, byte(c))
   602  				i += 2
   603  			case 'u', 'U':
   604  				t := p.s[i]
   605  				if !unicode {
   606  					return "", p.errorf("illegal escape sequence: \\%c", t)
   607  				}
   608  
   609  				i++
   610  				size := 4
   611  				if t == 'U' {
   612  					size = 8
   613  				}
   614  				if i+size-1 >= len(p.s) {
   615  					return "", p.errorf("illegal escape sequence: \\%c escape sequence must be followed by %d hex digits", t, size)
   616  				}
   617  				for j := 0; j < size; j++ {
   618  					if !isHexDigit(p.s[i+j]) {
   619  						return "", p.errorf("illegal escape sequence: \\%c escape sequence must be followed by %d hex digits", t, size)
   620  					}
   621  				}
   622  				c, err := strconv.ParseUint(p.s[i:i+size], 16, 64)
   623  				if err != nil {
   624  					return "", p.errorf("illegal escape sequence: invalid \\%c digits: %q: %v", t, p.s[i:i+size], err)
   625  				}
   626  				if 0xD800 <= c && c <= 0xDFFF || 0x10FFFF < c {
   627  					return "", p.errorf("illegal escape sequence: invalid codepoint: %x", c)
   628  				}
   629  				var buf [utf8.UTFMax]byte
   630  				n := utf8.EncodeRune(buf[:], rune(c))
   631  				content = append(content, buf[:n]...)
   632  				i += size
   633  			case '0', '1', '2', '3', '4', '5', '6', '7':
   634  				if !(i+2 < len(p.s) && isOctalDigit(p.s[i+1]) && isOctalDigit(p.s[i+2])) {
   635  					return "", p.errorf("illegal escape sequence: octal escape sequence must be followed by 3 octal digits")
   636  				}
   637  				c, err := strconv.ParseUint(p.s[i:i+3], 8, 64)
   638  				if err != nil {
   639  					return "", p.errorf("illegal escape sequence: invalid octal digits: %q: %v", p.s[i:i+3], err)
   640  				}
   641  				if c >= 256 {
   642  					return "", p.errorf("illegal escape sequence: octal digits overflow: %q (%d)", p.s[i:i+3], c)
   643  				}
   644  				content = append(content, byte(c))
   645  				i += 3
   646  			default:
   647  				return "", p.errorf("illegal escape sequence: \\%c", p.s[i])
   648  			}
   649  
   650  			continue
   651  		}
   652  
   653  		if p.s[i] == '\n' {
   654  			if len(delim) != 3 { // newline is only allowed inside triple-quoted.
   655  				return "", p.errorf("newline forbidden in %s", name)
   656  			}
   657  			p.line++
   658  		}
   659  
   660  		content = append(content, p.s[i])
   661  		i++
   662  	}
   663  
   664  	return "", p.errorf("unclosed %s", name)
   665  }
   666  
   667  var operators = map[string]bool{
   668  	// Arithmetic operators.
   669  	"-":  true, // both unary and binary
   670  	"~":  true,
   671  	"*":  true,
   672  	"/":  true,
   673  	"||": true,
   674  	"+":  true,
   675  	"<<": true,
   676  	">>": true,
   677  	"&":  true,
   678  	"^":  true,
   679  	"|":  true,
   680  
   681  	// Comparison operators.
   682  	"<":  true,
   683  	"<=": true,
   684  	">":  true,
   685  	">=": true,
   686  	"=":  true,
   687  	"!=": true,
   688  	"<>": true,
   689  }
   690  
   691  func isSpace(c byte) bool {
   692  	// Per https://cloud.google.com/spanner/docs/lexical, informally,
   693  	// whitespace is defined as "space, backspace, tab, newline".
   694  	switch c {
   695  	case ' ', '\b', '\t', '\n':
   696  		return true
   697  	}
   698  	return false
   699  }
   700  
   701  // skipSpace skips past any space or comments.
   702  func (p *parser) skipSpace() bool {
   703  	initLine := p.line
   704  	// If we start capturing a comment in this method,
   705  	// this is set to its comment value. Multi-line comments
   706  	// are only joined during a single skipSpace invocation.
   707  	var com *comment
   708  
   709  	i := 0
   710  	for i < len(p.s) {
   711  		if isSpace(p.s[i]) {
   712  			if p.s[i] == '\n' {
   713  				p.line++
   714  			}
   715  			i++
   716  			continue
   717  		}
   718  		// Comments.
   719  		marker, term := "", ""
   720  		if p.s[i] == '#' {
   721  			marker, term = "#", "\n"
   722  		} else if i+1 < len(p.s) && p.s[i] == '-' && p.s[i+1] == '-' {
   723  			marker, term = "--", "\n"
   724  		} else if i+1 < len(p.s) && p.s[i] == '/' && p.s[i+1] == '*' {
   725  			marker, term = "/*", "*/"
   726  		}
   727  		if term == "" {
   728  			break
   729  		}
   730  		// Search for the terminator, starting after the marker.
   731  		ti := strings.Index(p.s[i+len(marker):], term)
   732  		if ti < 0 {
   733  			p.errorf("unterminated comment")
   734  			return false
   735  		}
   736  		ti += len(marker) // make ti relative to p.s[i:]
   737  		if com != nil && (com.end.Line+1 < p.line || com.marker != marker) {
   738  			// There's a previous comment, but there's an
   739  			// intervening blank line, or the marker changed.
   740  			// Terminate the previous comment.
   741  			com = nil
   742  		}
   743  		if com == nil {
   744  			// New comment.
   745  			p.comments = append(p.comments, comment{
   746  				marker:   marker,
   747  				isolated: (p.line != initLine) || p.line == 1,
   748  				start: Position{
   749  					Line:   p.line,
   750  					Offset: p.offset + i,
   751  				},
   752  			})
   753  			com = &p.comments[len(p.comments)-1]
   754  		}
   755  		textLines := strings.Split(p.s[i+len(marker):i+ti], "\n")
   756  		com.text = append(com.text, textLines...)
   757  		com.end = Position{
   758  			Line:   p.line + len(textLines) - 1,
   759  			Offset: p.offset + i + ti,
   760  		}
   761  		p.line = com.end.Line
   762  		if term == "\n" {
   763  			p.line++
   764  		}
   765  		i += ti + len(term)
   766  
   767  		// A non-isolated comment is always complete and doesn't get
   768  		// combined with any future comment.
   769  		if !com.isolated {
   770  			com = nil
   771  		}
   772  	}
   773  	p.s = p.s[i:]
   774  	p.offset += i
   775  	if p.s == "" {
   776  		p.done = true
   777  	}
   778  	return i > 0
   779  }
   780  
   781  // advance moves the parser to the next token, which will be available in p.cur.
   782  func (p *parser) advance() {
   783  	prevID := p.cur.typ == quotedID || p.cur.typ == unquotedID
   784  
   785  	p.skipSpace()
   786  	if p.done {
   787  		return
   788  	}
   789  
   790  	// If the previous token was an identifier (quoted or unquoted),
   791  	// the next token being a dot means this is a path expression (not a number).
   792  	if prevID && p.s[0] == '.' {
   793  		p.cur.err = nil
   794  		p.cur.line, p.cur.offset = p.line, p.offset
   795  		p.cur.typ = unknownToken
   796  		p.cur.value, p.s = p.s[:1], p.s[1:]
   797  		p.offset++
   798  		return
   799  	}
   800  
   801  	p.cur.err = nil
   802  	p.cur.line, p.cur.offset = p.line, p.offset
   803  	p.cur.typ = unknownToken
   804  	// TODO: struct literals
   805  	switch p.s[0] {
   806  	case ',', ';', '(', ')', '{', '}', '[', ']', '*', '+', '-':
   807  		// Single character symbol.
   808  		p.cur.value, p.s = p.s[:1], p.s[1:]
   809  		p.offset++
   810  		return
   811  	// String literal prefix.
   812  	case 'B', 'b', 'R', 'r', '"', '\'':
   813  		// "B", "b", "BR", "Rb" etc are valid string literal prefix, however "BB", "rR" etc are not.
   814  		raw, bytes := false, false
   815  		for i := 0; i < 4 && i < len(p.s); i++ {
   816  			switch {
   817  			case !raw && (p.s[i] == 'R' || p.s[i] == 'r'):
   818  				raw = true
   819  				continue
   820  			case !bytes && (p.s[i] == 'B' || p.s[i] == 'b'):
   821  				bytes = true
   822  				continue
   823  			case p.s[i] == '"' || p.s[i] == '\'':
   824  				switch {
   825  				case raw && bytes:
   826  					p.consumeRawBytes()
   827  				case raw:
   828  					p.consumeRawString()
   829  				case bytes:
   830  					p.consumeBytes()
   831  				default:
   832  					p.consumeString()
   833  				}
   834  				return
   835  			}
   836  			break
   837  		}
   838  	case '`':
   839  		// Quoted identifier.
   840  		p.cur.string, p.cur.err = p.consumeStringContent("`", false, true, "quoted identifier")
   841  		p.cur.typ = quotedID
   842  		return
   843  	}
   844  	if p.s[0] == '@' || isInitialIdentifierChar(p.s[0]) {
   845  		// Start consuming identifier.
   846  		i := 1
   847  		for i < len(p.s) && isIdentifierChar(p.s[i]) {
   848  			i++
   849  		}
   850  		p.cur.value, p.s = p.s[:i], p.s[i:]
   851  		p.cur.typ = unquotedID
   852  		p.offset += i
   853  		return
   854  	}
   855  	if len(p.s) >= 2 && p.s[0] == '.' && ('0' <= p.s[1] && p.s[1] <= '9') {
   856  		// dot followed by a digit.
   857  		p.consumeNumber()
   858  		return
   859  	}
   860  	if '0' <= p.s[0] && p.s[0] <= '9' {
   861  		p.consumeNumber()
   862  		return
   863  	}
   864  
   865  	// Look for operator (two or one bytes).
   866  	for i := 2; i >= 1; i-- {
   867  		if i <= len(p.s) && operators[p.s[:i]] {
   868  			p.cur.value, p.s = p.s[:i], p.s[i:]
   869  			p.offset += i
   870  			return
   871  		}
   872  	}
   873  
   874  	p.errorf("unexpected byte %#x", p.s[0])
   875  }
   876  
   877  // back steps the parser back one token. It cannot be called twice in succession.
   878  func (p *parser) back() {
   879  	if p.backed {
   880  		panic("parser backed up twice")
   881  	}
   882  	p.done = false
   883  	p.backed = true
   884  	// If an error was being recovered, we wish to ignore the error.
   885  	// Don't do that for eof since that'll be returned next.
   886  	if p.cur.err != eof {
   887  		p.cur.err = nil
   888  	}
   889  }
   890  
   891  // next returns the next token.
   892  func (p *parser) next() *token {
   893  	if p.backed || p.done {
   894  		p.backed = false
   895  		return &p.cur
   896  	}
   897  	p.advance()
   898  	if p.done && p.cur.err == nil {
   899  		p.cur.value = ""
   900  		p.cur.err = eof
   901  	}
   902  	debugf("parser·next(): returning [%v] [err: %v] @l%d,o%d", p.cur.value, p.cur.err, p.cur.line, p.cur.offset)
   903  	return &p.cur
   904  }
   905  
   906  // caseEqual reports whether the token is valid, not a quoted identifier, and
   907  // equal to the provided string under a case insensitive comparison.
   908  // Use this (or sniff/eat/expect) instead of comparing a string directly for keywords, etc.
   909  func (t *token) caseEqual(x string) bool {
   910  	return t.err == nil && t.typ != quotedID && strings.EqualFold(t.value, x)
   911  }
   912  
   913  // sniff reports whether the next N tokens are as specified.
   914  func (p *parser) sniff(want ...string) bool {
   915  	// Store current parser state and restore on the way out.
   916  	orig := *p
   917  	defer func() { *p = orig }()
   918  
   919  	for _, w := range want {
   920  		if !p.next().caseEqual(w) {
   921  			return false
   922  		}
   923  	}
   924  	return true
   925  }
   926  
   927  // sniffTokenType reports whether the next token type is as specified.
   928  func (p *parser) sniffTokenType(want tokenType) bool {
   929  	orig := *p
   930  	defer func() { *p = orig }()
   931  
   932  	if p.next().typ == want {
   933  		return true
   934  	}
   935  	return false
   936  }
   937  
   938  // eat reports whether the next N tokens are as specified,
   939  // then consumes them.
   940  func (p *parser) eat(want ...string) bool {
   941  	// Store current parser state so we can restore if we get a failure.
   942  	orig := *p
   943  
   944  	for _, w := range want {
   945  		if !p.next().caseEqual(w) {
   946  			// Mismatch.
   947  			*p = orig
   948  			return false
   949  		}
   950  	}
   951  	return true
   952  }
   953  
   954  func (p *parser) expect(want ...string) *parseError {
   955  	for _, w := range want {
   956  		tok := p.next()
   957  		if tok.err != nil {
   958  			return tok.err
   959  		}
   960  		if !tok.caseEqual(w) {
   961  			return p.errorf("got %q while expecting %q", tok.value, w)
   962  		}
   963  	}
   964  	return nil
   965  }
   966  
   967  func (p *parser) parseDDLStmt() (DDLStmt, *parseError) {
   968  	debugf("parseDDLStmt: %v", p)
   969  
   970  	/*
   971  		statement:
   972  			{ create_database | create_table | create_index | alter_table | drop_table | drop_index }
   973  	*/
   974  
   975  	// TODO: support create_database
   976  
   977  	if p.sniff("CREATE", "TABLE") {
   978  		ct, err := p.parseCreateTable()
   979  		return ct, err
   980  	} else if p.sniff("CREATE", "INDEX") || p.sniff("CREATE", "UNIQUE", "INDEX") || p.sniff("CREATE", "NULL_FILTERED", "INDEX") || p.sniff("CREATE", "UNIQUE", "NULL_FILTERED", "INDEX") {
   981  		ci, err := p.parseCreateIndex()
   982  		return ci, err
   983  	} else if p.sniff("CREATE", "VIEW") || p.sniff("CREATE", "OR", "REPLACE", "VIEW") {
   984  		cv, err := p.parseCreateView()
   985  		return cv, err
   986  	} else if p.sniff("ALTER", "TABLE") {
   987  		a, err := p.parseAlterTable()
   988  		return a, err
   989  	} else if p.eat("DROP") {
   990  		pos := p.Pos()
   991  		// These statements are simple.
   992  		// DROP TABLE table_name
   993  		// DROP INDEX index_name
   994  		// DROP VIEW view_name
   995  		tok := p.next()
   996  		if tok.err != nil {
   997  			return nil, tok.err
   998  		}
   999  		switch {
  1000  		default:
  1001  			return nil, p.errorf("got %q, want TABLE, VIEW or INDEX", tok.value)
  1002  		case tok.caseEqual("TABLE"):
  1003  			name, err := p.parseTableOrIndexOrColumnName()
  1004  			if err != nil {
  1005  				return nil, err
  1006  			}
  1007  			return &DropTable{Name: name, Position: pos}, nil
  1008  		case tok.caseEqual("INDEX"):
  1009  			name, err := p.parseTableOrIndexOrColumnName()
  1010  			if err != nil {
  1011  				return nil, err
  1012  			}
  1013  			return &DropIndex{Name: name, Position: pos}, nil
  1014  		case tok.caseEqual("VIEW"):
  1015  			name, err := p.parseTableOrIndexOrColumnName()
  1016  			if err != nil {
  1017  				return nil, err
  1018  			}
  1019  			return &DropView{Name: name, Position: pos}, nil
  1020  		}
  1021  	} else if p.sniff("ALTER", "DATABASE") {
  1022  		a, err := p.parseAlterDatabase()
  1023  		return a, err
  1024  	}
  1025  
  1026  	return nil, unknownDDL
  1027  }
  1028  
  1029  func (p *parser) parseCreateTable() (*CreateTable, *parseError) {
  1030  	debugf("parseCreateTable: %v", p)
  1031  
  1032  	/*
  1033  		CREATE TABLE table_name(
  1034  			[column_def, ...] [ table_constraint, ...] )
  1035  			primary_key [, cluster]
  1036  
  1037  		primary_key:
  1038  			PRIMARY KEY ( [key_part, ...] )
  1039  
  1040  		cluster:
  1041  			INTERLEAVE IN PARENT table_name [ ON DELETE { CASCADE | NO ACTION } ]
  1042  	*/
  1043  
  1044  	if err := p.expect("CREATE"); err != nil {
  1045  		return nil, err
  1046  	}
  1047  	pos := p.Pos()
  1048  	if err := p.expect("TABLE"); err != nil {
  1049  		return nil, err
  1050  	}
  1051  	tname, err := p.parseTableOrIndexOrColumnName()
  1052  	if err != nil {
  1053  		return nil, err
  1054  	}
  1055  
  1056  	ct := &CreateTable{Name: tname, Position: pos}
  1057  	err = p.parseCommaList("(", ")", func(p *parser) *parseError {
  1058  		if p.sniffTableConstraint() {
  1059  			tc, err := p.parseTableConstraint()
  1060  			if err != nil {
  1061  				return err
  1062  			}
  1063  			ct.Constraints = append(ct.Constraints, tc)
  1064  			return nil
  1065  		}
  1066  
  1067  		cd, err := p.parseColumnDef()
  1068  		if err != nil {
  1069  			return err
  1070  		}
  1071  		ct.Columns = append(ct.Columns, cd)
  1072  		return nil
  1073  	})
  1074  	if err != nil {
  1075  		return nil, err
  1076  	}
  1077  
  1078  	if err := p.expect("PRIMARY"); err != nil {
  1079  		return nil, err
  1080  	}
  1081  	if err := p.expect("KEY"); err != nil {
  1082  		return nil, err
  1083  	}
  1084  	ct.PrimaryKey, err = p.parseKeyPartList()
  1085  	if err != nil {
  1086  		return nil, err
  1087  	}
  1088  
  1089  	if p.eat(",", "INTERLEAVE") {
  1090  		if err := p.expect("IN"); err != nil {
  1091  			return nil, err
  1092  		}
  1093  		if err := p.expect("PARENT"); err != nil {
  1094  			return nil, err
  1095  		}
  1096  		pname, err := p.parseTableOrIndexOrColumnName()
  1097  		if err != nil {
  1098  			return nil, err
  1099  		}
  1100  		ct.Interleave = &Interleave{
  1101  			Parent:   pname,
  1102  			OnDelete: NoActionOnDelete,
  1103  		}
  1104  		// The ON DELETE clause is optional; it defaults to NoActionOnDelete.
  1105  		if p.eat("ON", "DELETE") {
  1106  			od, err := p.parseOnDelete()
  1107  			if err != nil {
  1108  				return nil, err
  1109  			}
  1110  			ct.Interleave.OnDelete = od
  1111  		}
  1112  	}
  1113  	if p.eat(",", "ROW", "DELETION", "POLICY") {
  1114  		rdp, err := p.parseRowDeletionPolicy()
  1115  		if err != nil {
  1116  			return nil, err
  1117  		}
  1118  		ct.RowDeletionPolicy = &rdp
  1119  	}
  1120  
  1121  	return ct, nil
  1122  }
  1123  
  1124  func (p *parser) sniffTableConstraint() bool {
  1125  	// Unfortunately the Cloud Spanner grammar is LL(3) because
  1126  	//	CONSTRAINT BOOL
  1127  	// could be the start of a declaration of a column called "CONSTRAINT" of boolean type,
  1128  	// or it could be the start of a foreign key constraint called "BOOL".
  1129  	// We have to sniff up to the third token to see what production it is.
  1130  	// If we have "FOREIGN" and "KEY" (or "CHECK"), this is an unnamed table constraint.
  1131  	// If we have "CONSTRAINT", an identifier and "FOREIGN" (or "CHECK"), this is a table constraint.
  1132  	// Otherwise, this is a column definition.
  1133  
  1134  	if p.sniff("FOREIGN", "KEY") || p.sniff("CHECK") {
  1135  		return true
  1136  	}
  1137  
  1138  	// Store parser state, and peek ahead.
  1139  	// Restore on the way out.
  1140  	orig := *p
  1141  	defer func() { *p = orig }()
  1142  
  1143  	if !p.eat("CONSTRAINT") {
  1144  		return false
  1145  	}
  1146  	if _, err := p.parseTableOrIndexOrColumnName(); err != nil {
  1147  		return false
  1148  	}
  1149  	return p.sniff("FOREIGN") || p.sniff("CHECK")
  1150  }
  1151  
  1152  func (p *parser) parseCreateIndex() (*CreateIndex, *parseError) {
  1153  	debugf("parseCreateIndex: %v", p)
  1154  
  1155  	/*
  1156  		CREATE [UNIQUE] [NULL_FILTERED] INDEX index_name
  1157  			ON table_name ( key_part [, ...] ) [ storing_clause ] [ , interleave_clause ]
  1158  
  1159  		index_name:
  1160  			{a—z|A—Z}[{a—z|A—Z|0—9|_}+]
  1161  
  1162  		storing_clause:
  1163  			STORING ( column_name [, ...] )
  1164  
  1165  		interleave_clause:
  1166  			INTERLEAVE IN table_name
  1167  	*/
  1168  
  1169  	var unique, nullFiltered bool
  1170  
  1171  	if err := p.expect("CREATE"); err != nil {
  1172  		return nil, err
  1173  	}
  1174  	pos := p.Pos()
  1175  	if p.eat("UNIQUE") {
  1176  		unique = true
  1177  	}
  1178  	if p.eat("NULL_FILTERED") {
  1179  		nullFiltered = true
  1180  	}
  1181  	if err := p.expect("INDEX"); err != nil {
  1182  		return nil, err
  1183  	}
  1184  	iname, err := p.parseTableOrIndexOrColumnName()
  1185  	if err != nil {
  1186  		return nil, err
  1187  	}
  1188  	if err := p.expect("ON"); err != nil {
  1189  		return nil, err
  1190  	}
  1191  	tname, err := p.parseTableOrIndexOrColumnName()
  1192  	if err != nil {
  1193  		return nil, err
  1194  	}
  1195  	ci := &CreateIndex{
  1196  		Name:  iname,
  1197  		Table: tname,
  1198  
  1199  		Unique:       unique,
  1200  		NullFiltered: nullFiltered,
  1201  
  1202  		Position: pos,
  1203  	}
  1204  	ci.Columns, err = p.parseKeyPartList()
  1205  	if err != nil {
  1206  		return nil, err
  1207  	}
  1208  
  1209  	if p.eat("STORING") {
  1210  		ci.Storing, err = p.parseColumnNameList()
  1211  		if err != nil {
  1212  			return nil, err
  1213  		}
  1214  	}
  1215  
  1216  	if p.eat(",", "INTERLEAVE", "IN") {
  1217  		ci.Interleave, err = p.parseTableOrIndexOrColumnName()
  1218  		if err != nil {
  1219  			return nil, err
  1220  		}
  1221  	}
  1222  
  1223  	return ci, nil
  1224  }
  1225  
  1226  func (p *parser) parseCreateView() (*CreateView, *parseError) {
  1227  	debugf("parseCreateView: %v", p)
  1228  
  1229  	/*
  1230  		{ CREATE VIEW | CREATE OR REPLACE VIEW } view_name
  1231  		SQL SECURITY INVOKER
  1232  		AS query
  1233  	*/
  1234  
  1235  	var orReplace bool
  1236  
  1237  	if err := p.expect("CREATE"); err != nil {
  1238  		return nil, err
  1239  	}
  1240  	pos := p.Pos()
  1241  	if p.eat("OR", "REPLACE") {
  1242  		orReplace = true
  1243  	}
  1244  	if err := p.expect("VIEW"); err != nil {
  1245  		return nil, err
  1246  	}
  1247  	vname, err := p.parseTableOrIndexOrColumnName()
  1248  	if err := p.expect("SQL", "SECURITY", "INVOKER", "AS"); err != nil {
  1249  		return nil, err
  1250  	}
  1251  	query, err := p.parseQuery()
  1252  	if err != nil {
  1253  		return nil, err
  1254  	}
  1255  
  1256  	return &CreateView{
  1257  		Name:      vname,
  1258  		OrReplace: orReplace,
  1259  		Query:     query,
  1260  
  1261  		Position: pos,
  1262  	}, nil
  1263  }
  1264  
  1265  func (p *parser) parseAlterTable() (*AlterTable, *parseError) {
  1266  	debugf("parseAlterTable: %v", p)
  1267  
  1268  	/*
  1269  		alter_table:
  1270  			ALTER TABLE table_name { table_alteration | table_column_alteration }
  1271  
  1272  		table_alteration:
  1273  			{ ADD [ COLUMN ] column_def
  1274  			| DROP [ COLUMN ] column_name
  1275  			| ADD table_constraint
  1276  			| DROP CONSTRAINT constraint_name
  1277  			| SET ON DELETE { CASCADE | NO ACTION } }
  1278  
  1279  		table_column_alteration:
  1280  			ALTER [ COLUMN ] column_name { { scalar_type | array_type } [NOT NULL] | SET options_def }
  1281  	*/
  1282  
  1283  	if err := p.expect("ALTER"); err != nil {
  1284  		return nil, err
  1285  	}
  1286  	pos := p.Pos()
  1287  	if err := p.expect("TABLE"); err != nil {
  1288  		return nil, err
  1289  	}
  1290  	tname, err := p.parseTableOrIndexOrColumnName()
  1291  	if err != nil {
  1292  		return nil, err
  1293  	}
  1294  	a := &AlterTable{Name: tname, Position: pos}
  1295  
  1296  	tok := p.next()
  1297  	if tok.err != nil {
  1298  		return nil, tok.err
  1299  	}
  1300  	switch {
  1301  	default:
  1302  		return nil, p.errorf("got %q, expected ADD or DROP or SET or ALTER", tok.value)
  1303  	case tok.caseEqual("ADD"):
  1304  		if p.sniff("CONSTRAINT") || p.sniff("FOREIGN") || p.sniff("CHECK") {
  1305  			tc, err := p.parseTableConstraint()
  1306  			if err != nil {
  1307  				return nil, err
  1308  			}
  1309  			a.Alteration = AddConstraint{Constraint: tc}
  1310  			return a, nil
  1311  		}
  1312  
  1313  		if p.eat("ROW", "DELETION", "POLICY") {
  1314  			rdp, err := p.parseRowDeletionPolicy()
  1315  			if err != nil {
  1316  				return nil, err
  1317  			}
  1318  			a.Alteration = AddRowDeletionPolicy{RowDeletionPolicy: rdp}
  1319  			return a, nil
  1320  		}
  1321  
  1322  		// TODO: "COLUMN" is optional.
  1323  		if err := p.expect("COLUMN"); err != nil {
  1324  			return nil, err
  1325  		}
  1326  		cd, err := p.parseColumnDef()
  1327  		if err != nil {
  1328  			return nil, err
  1329  		}
  1330  		a.Alteration = AddColumn{Def: cd}
  1331  		return a, nil
  1332  	case tok.caseEqual("DROP"):
  1333  		if p.eat("CONSTRAINT") {
  1334  			name, err := p.parseTableOrIndexOrColumnName()
  1335  			if err != nil {
  1336  				return nil, err
  1337  			}
  1338  			a.Alteration = DropConstraint{Name: name}
  1339  			return a, nil
  1340  		}
  1341  
  1342  		if p.eat("ROW", "DELETION", "POLICY") {
  1343  			a.Alteration = DropRowDeletionPolicy{}
  1344  			return a, nil
  1345  		}
  1346  
  1347  		// TODO: "COLUMN" is optional.
  1348  		if err := p.expect("COLUMN"); err != nil {
  1349  			return nil, err
  1350  		}
  1351  		name, err := p.parseTableOrIndexOrColumnName()
  1352  		if err != nil {
  1353  			return nil, err
  1354  		}
  1355  		a.Alteration = DropColumn{Name: name}
  1356  		return a, nil
  1357  	case tok.caseEqual("SET"):
  1358  		if err := p.expect("ON"); err != nil {
  1359  			return nil, err
  1360  		}
  1361  		if err := p.expect("DELETE"); err != nil {
  1362  			return nil, err
  1363  		}
  1364  		od, err := p.parseOnDelete()
  1365  		if err != nil {
  1366  			return nil, err
  1367  		}
  1368  		a.Alteration = SetOnDelete{Action: od}
  1369  		return a, nil
  1370  	case tok.caseEqual("ALTER"):
  1371  		// TODO: "COLUMN" is optional.
  1372  		if err := p.expect("COLUMN"); err != nil {
  1373  			return nil, err
  1374  		}
  1375  		name, err := p.parseTableOrIndexOrColumnName()
  1376  		if err != nil {
  1377  			return nil, err
  1378  		}
  1379  		ca, err := p.parseColumnAlteration()
  1380  		if err != nil {
  1381  			return nil, err
  1382  		}
  1383  		a.Alteration = AlterColumn{
  1384  			Name:       name,
  1385  			Alteration: ca,
  1386  		}
  1387  		return a, nil
  1388  	case tok.caseEqual("REPLACE"):
  1389  		if p.eat("ROW", "DELETION", "POLICY") {
  1390  			rdp, err := p.parseRowDeletionPolicy()
  1391  			if err != nil {
  1392  				return nil, err
  1393  			}
  1394  			a.Alteration = ReplaceRowDeletionPolicy{RowDeletionPolicy: rdp}
  1395  			return a, nil
  1396  		}
  1397  	}
  1398  	return a, nil
  1399  }
  1400  
  1401  func (p *parser) parseAlterDatabase() (*AlterDatabase, *parseError) {
  1402  	debugf("parseAlterDatabase: %v", p)
  1403  
  1404  	/*
  1405  		ALTER DATABASE database_id
  1406  			action
  1407  
  1408  		where database_id is:
  1409  			{a—z}[{a—z|0—9|_|-}+]{a—z|0—9}
  1410  
  1411  		and action is:
  1412  			SET OPTIONS ( optimizer_version = { 1 ...  2 | null },
  1413  						  version_retention_period = { 'duration' | null } )
  1414  	*/
  1415  
  1416  	if err := p.expect("ALTER"); err != nil {
  1417  		return nil, err
  1418  	}
  1419  	pos := p.Pos()
  1420  	if err := p.expect("DATABASE"); err != nil {
  1421  		return nil, err
  1422  	}
  1423  	// This is not 100% correct as database identifiers have slightly more
  1424  	// restrictions than table names, but the restrictions are currently not
  1425  	// applied in the spansql parser.
  1426  	// TODO: Apply restrictions for all identifiers.
  1427  	dbname, err := p.parseTableOrIndexOrColumnName()
  1428  	if err != nil {
  1429  		return nil, err
  1430  	}
  1431  	a := &AlterDatabase{Name: dbname, Position: pos}
  1432  
  1433  	tok := p.next()
  1434  	if tok.err != nil {
  1435  		return nil, tok.err
  1436  	}
  1437  	switch {
  1438  	default:
  1439  		return nil, p.errorf("got %q, expected SET", tok.value)
  1440  	case tok.caseEqual("SET"):
  1441  		options, err := p.parseDatabaseOptions()
  1442  		if err != nil {
  1443  			return nil, err
  1444  		}
  1445  		a.Alteration = SetDatabaseOptions{Options: options}
  1446  		return a, nil
  1447  	}
  1448  }
  1449  
  1450  func (p *parser) parseDMLStmt() (DMLStmt, *parseError) {
  1451  	debugf("parseDMLStmt: %v", p)
  1452  
  1453  	/*
  1454  		DELETE [FROM] target_name [[AS] alias]
  1455  		WHERE condition
  1456  
  1457  		UPDATE target_name [[AS] alias]
  1458  		SET update_item [, ...]
  1459  		WHERE condition
  1460  
  1461  		update_item: path_expression = expression | path_expression = DEFAULT
  1462  
  1463  		INSERT [INTO] target_name
  1464  		 (column_name_1 [, ..., column_name_n] )
  1465  		 input
  1466  
  1467  		input:
  1468  		 VALUES (row_1_column_1_expr [, ..., row_1_column_n_expr ] )
  1469  		        [, ..., (row_k_column_1_expr [, ..., row_k_column_n_expr ] ) ]
  1470  		| select_query
  1471  
  1472  		expr: value_expression | DEFAULT
  1473  	*/
  1474  
  1475  	if p.eat("DELETE") {
  1476  		p.eat("FROM") // optional
  1477  		tname, err := p.parseTableOrIndexOrColumnName()
  1478  		if err != nil {
  1479  			return nil, err
  1480  		}
  1481  		// TODO: parse alias.
  1482  		if err := p.expect("WHERE"); err != nil {
  1483  			return nil, err
  1484  		}
  1485  		where, err := p.parseBoolExpr()
  1486  		if err != nil {
  1487  			return nil, err
  1488  		}
  1489  		return &Delete{
  1490  			Table: tname,
  1491  			Where: where,
  1492  		}, nil
  1493  	}
  1494  
  1495  	if p.eat("UPDATE") {
  1496  		tname, err := p.parseTableOrIndexOrColumnName()
  1497  		if err != nil {
  1498  			return nil, err
  1499  		}
  1500  		u := &Update{
  1501  			Table: tname,
  1502  		}
  1503  		// TODO: parse alias.
  1504  		if err := p.expect("SET"); err != nil {
  1505  			return nil, err
  1506  		}
  1507  		for {
  1508  			ui, err := p.parseUpdateItem()
  1509  			if err != nil {
  1510  				return nil, err
  1511  			}
  1512  			u.Items = append(u.Items, ui)
  1513  			if p.eat(",") {
  1514  				continue
  1515  			}
  1516  			break
  1517  		}
  1518  		if err := p.expect("WHERE"); err != nil {
  1519  			return nil, err
  1520  		}
  1521  		where, err := p.parseBoolExpr()
  1522  		if err != nil {
  1523  			return nil, err
  1524  		}
  1525  		u.Where = where
  1526  		return u, nil
  1527  	}
  1528  
  1529  	if p.eat("INSERT") {
  1530  		p.eat("INTO") // optional
  1531  		tname, err := p.parseTableOrIndexOrColumnName()
  1532  		if err != nil {
  1533  			return nil, err
  1534  		}
  1535  
  1536  		columns, err := p.parseColumnNameList()
  1537  		if err != nil {
  1538  			return nil, err
  1539  		}
  1540  
  1541  		var input ValuesOrSelect
  1542  		if p.eat("VALUES") {
  1543  			values := make([][]Expr, 0)
  1544  			for {
  1545  				exprs, err := p.parseParenExprList()
  1546  				if err != nil {
  1547  					return nil, err
  1548  				}
  1549  				values = append(values, exprs)
  1550  				if !p.eat(",") {
  1551  					break
  1552  				}
  1553  			}
  1554  			input = Values(values)
  1555  		} else {
  1556  			input, err = p.parseSelect()
  1557  			if err != nil {
  1558  				return nil, err
  1559  			}
  1560  		}
  1561  
  1562  		return &Insert{
  1563  			Table:   tname,
  1564  			Columns: columns,
  1565  			Input:   input,
  1566  		}, nil
  1567  	}
  1568  
  1569  	return nil, p.errorf("unknown DML statement")
  1570  }
  1571  
  1572  func (p *parser) parseUpdateItem() (UpdateItem, *parseError) {
  1573  	col, err := p.parseTableOrIndexOrColumnName()
  1574  	if err != nil {
  1575  		return UpdateItem{}, err
  1576  	}
  1577  	ui := UpdateItem{
  1578  		Column: col,
  1579  	}
  1580  	if err := p.expect("="); err != nil {
  1581  		return UpdateItem{}, err
  1582  	}
  1583  	if p.eat("DEFAULT") {
  1584  		return ui, nil
  1585  	}
  1586  	ui.Value, err = p.parseExpr()
  1587  	if err != nil {
  1588  		return UpdateItem{}, err
  1589  	}
  1590  	return ui, nil
  1591  }
  1592  
  1593  func (p *parser) parseColumnDef() (ColumnDef, *parseError) {
  1594  	debugf("parseColumnDef: %v", p)
  1595  
  1596  	/*
  1597  		column_def:
  1598  			column_name {scalar_type | array_type} [NOT NULL] [{DEFAULT ( expression ) | AS ( expression ) STORED}] [options_def]
  1599  	*/
  1600  
  1601  	name, err := p.parseTableOrIndexOrColumnName()
  1602  	if err != nil {
  1603  		return ColumnDef{}, err
  1604  	}
  1605  
  1606  	cd := ColumnDef{Name: name, Position: p.Pos()}
  1607  
  1608  	cd.Type, err = p.parseType()
  1609  	if err != nil {
  1610  		return ColumnDef{}, err
  1611  	}
  1612  
  1613  	if p.eat("NOT", "NULL") {
  1614  		cd.NotNull = true
  1615  	}
  1616  
  1617  	if p.eat("DEFAULT", "(") {
  1618  		cd.Default, err = p.parseExpr()
  1619  		if err != nil {
  1620  			return ColumnDef{}, err
  1621  		}
  1622  		if err := p.expect(")"); err != nil {
  1623  			return ColumnDef{}, err
  1624  		}
  1625  	}
  1626  
  1627  	if p.eat("AS", "(") {
  1628  		cd.Generated, err = p.parseExpr()
  1629  		if err != nil {
  1630  			return ColumnDef{}, err
  1631  		}
  1632  		if err := p.expect(")"); err != nil {
  1633  			return ColumnDef{}, err
  1634  		}
  1635  		if err := p.expect("STORED"); err != nil {
  1636  			return ColumnDef{}, err
  1637  		}
  1638  	}
  1639  
  1640  	if p.sniff("OPTIONS") {
  1641  		cd.Options, err = p.parseColumnOptions()
  1642  		if err != nil {
  1643  			return ColumnDef{}, err
  1644  		}
  1645  	}
  1646  
  1647  	return cd, nil
  1648  }
  1649  
  1650  func (p *parser) parseColumnAlteration() (ColumnAlteration, *parseError) {
  1651  	debugf("parseColumnAlteration: %v", p)
  1652  	/*
  1653  		{
  1654  			data_type [ NOT NULL ] [ DEFAULT ( expression ) ]
  1655  			| SET  ( options_def )
  1656  			| SET  DEFAULT ( expression )
  1657  			| DROP DEFAULT
  1658  		}
  1659  	*/
  1660  
  1661  	if p.eat("SET", "DEFAULT", "(") {
  1662  		d, err := p.parseExpr()
  1663  		if err != nil {
  1664  			return nil, err
  1665  		}
  1666  		if err := p.expect(")"); err != nil {
  1667  			return nil, err
  1668  		}
  1669  		return SetDefault{Default: d}, nil
  1670  	}
  1671  
  1672  	if p.eat("DROP", "DEFAULT") {
  1673  		return DropDefault{}, nil
  1674  	}
  1675  
  1676  	if p.eat("SET") {
  1677  		co, err := p.parseColumnOptions()
  1678  		if err != nil {
  1679  			return nil, err
  1680  		}
  1681  		return SetColumnOptions{Options: co}, nil
  1682  	}
  1683  
  1684  	typ, err := p.parseType()
  1685  	if err != nil {
  1686  		return nil, err
  1687  	}
  1688  	sct := SetColumnType{Type: typ}
  1689  
  1690  	if p.eat("NOT", "NULL") {
  1691  		sct.NotNull = true
  1692  	}
  1693  
  1694  	if p.eat("DEFAULT", "(") {
  1695  		sct.Default, err = p.parseExpr()
  1696  		if err != nil {
  1697  			return nil, err
  1698  		}
  1699  		if err := p.expect(")"); err != nil {
  1700  			return nil, err
  1701  		}
  1702  	}
  1703  
  1704  	return sct, nil
  1705  }
  1706  
  1707  func (p *parser) parseColumnOptions() (ColumnOptions, *parseError) {
  1708  	debugf("parseColumnOptions: %v", p)
  1709  	/*
  1710  		options_def:
  1711  			OPTIONS (allow_commit_timestamp = { true | null })
  1712  	*/
  1713  
  1714  	if err := p.expect("OPTIONS"); err != nil {
  1715  		return ColumnOptions{}, err
  1716  	}
  1717  	if err := p.expect("("); err != nil {
  1718  		return ColumnOptions{}, err
  1719  	}
  1720  
  1721  	// TODO: Figure out if column options are case insensitive.
  1722  	// We ignore case for the key (because it is easier) but not the value.
  1723  	var co ColumnOptions
  1724  	if p.eat("allow_commit_timestamp", "=") {
  1725  		tok := p.next()
  1726  		if tok.err != nil {
  1727  			return ColumnOptions{}, tok.err
  1728  		}
  1729  		allowCommitTimestamp := new(bool)
  1730  		switch tok.value {
  1731  		case "true":
  1732  			*allowCommitTimestamp = true
  1733  		case "null":
  1734  			*allowCommitTimestamp = false
  1735  		default:
  1736  			return ColumnOptions{}, p.errorf("got %q, want true or null", tok.value)
  1737  		}
  1738  		co.AllowCommitTimestamp = allowCommitTimestamp
  1739  	}
  1740  
  1741  	if err := p.expect(")"); err != nil {
  1742  		return ColumnOptions{}, err
  1743  	}
  1744  
  1745  	return co, nil
  1746  }
  1747  
  1748  func (p *parser) parseDatabaseOptions() (DatabaseOptions, *parseError) {
  1749  	debugf("parseDatabaseOptions: %v", p)
  1750  	/*
  1751  		options_def:
  1752  			OPTIONS (enable_key_visualizer = { true | null },
  1753  					 optimizer_version = { 1 ... 2 | null },
  1754  					 version_retention_period = { 'duration' | null })
  1755  	*/
  1756  
  1757  	if err := p.expect("OPTIONS"); err != nil {
  1758  		return DatabaseOptions{}, err
  1759  	}
  1760  	if err := p.expect("("); err != nil {
  1761  		return DatabaseOptions{}, err
  1762  	}
  1763  
  1764  	// We ignore case for the key (because it is easier) but not the value.
  1765  	var opts DatabaseOptions
  1766  	for {
  1767  		if p.eat("enable_key_visualizer", "=") {
  1768  			tok := p.next()
  1769  			if tok.err != nil {
  1770  				return DatabaseOptions{}, tok.err
  1771  			}
  1772  			enableKeyVisualizer := new(bool)
  1773  			switch tok.value {
  1774  			case "true":
  1775  				*enableKeyVisualizer = true
  1776  			case "null":
  1777  				*enableKeyVisualizer = false
  1778  			default:
  1779  				return DatabaseOptions{}, p.errorf("invalid enable_key_visualizer_value: %v", tok.value)
  1780  			}
  1781  			opts.EnableKeyVisualizer = enableKeyVisualizer
  1782  		} else if p.eat("optimizer_version", "=") {
  1783  			tok := p.next()
  1784  			if tok.err != nil {
  1785  				return DatabaseOptions{}, tok.err
  1786  			}
  1787  			optimizerVersion := new(int)
  1788  			if tok.value == "null" {
  1789  				*optimizerVersion = 0
  1790  			} else {
  1791  				if tok.typ != int64Token {
  1792  					return DatabaseOptions{}, p.errorf("invalid optimizer_version value: %v", tok.value)
  1793  				}
  1794  				version, err := strconv.Atoi(tok.value)
  1795  				if err != nil {
  1796  					return DatabaseOptions{}, p.errorf("invalid optimizer_version value: %v", tok.value)
  1797  				}
  1798  				*optimizerVersion = version
  1799  			}
  1800  			opts.OptimizerVersion = optimizerVersion
  1801  		} else if p.eat("version_retention_period", "=") {
  1802  			tok := p.next()
  1803  			if tok.err != nil {
  1804  				return DatabaseOptions{}, tok.err
  1805  			}
  1806  			retentionPeriod := new(string)
  1807  			if tok.value == "null" {
  1808  				*retentionPeriod = ""
  1809  			} else {
  1810  				if tok.typ != stringToken {
  1811  					return DatabaseOptions{}, p.errorf("invalid version_retention_period: %v", tok.value)
  1812  				}
  1813  				*retentionPeriod = tok.string
  1814  			}
  1815  			opts.VersionRetentionPeriod = retentionPeriod
  1816  		} else {
  1817  			tok := p.next()
  1818  			return DatabaseOptions{}, p.errorf("unknown database option: %v", tok.value)
  1819  		}
  1820  		if p.sniff(")") {
  1821  			break
  1822  		}
  1823  		if !p.eat(",") {
  1824  			return DatabaseOptions{}, p.errorf("missing ',' in options list")
  1825  		}
  1826  	}
  1827  	if err := p.expect(")"); err != nil {
  1828  		return DatabaseOptions{}, err
  1829  	}
  1830  
  1831  	return opts, nil
  1832  }
  1833  
  1834  func (p *parser) parseKeyPartList() ([]KeyPart, *parseError) {
  1835  	var list []KeyPart
  1836  	err := p.parseCommaList("(", ")", func(p *parser) *parseError {
  1837  		kp, err := p.parseKeyPart()
  1838  		if err != nil {
  1839  			return err
  1840  		}
  1841  		list = append(list, kp)
  1842  		return nil
  1843  	})
  1844  	return list, err
  1845  }
  1846  
  1847  func (p *parser) parseKeyPart() (KeyPart, *parseError) {
  1848  	debugf("parseKeyPart: %v", p)
  1849  
  1850  	/*
  1851  		key_part:
  1852  			column_name [{ ASC | DESC }]
  1853  	*/
  1854  
  1855  	name, err := p.parseTableOrIndexOrColumnName()
  1856  	if err != nil {
  1857  		return KeyPart{}, err
  1858  	}
  1859  
  1860  	kp := KeyPart{Column: name}
  1861  
  1862  	if p.eat("ASC") {
  1863  		// OK.
  1864  	} else if p.eat("DESC") {
  1865  		kp.Desc = true
  1866  	}
  1867  
  1868  	return kp, nil
  1869  }
  1870  
  1871  func (p *parser) parseTableConstraint() (TableConstraint, *parseError) {
  1872  	debugf("parseTableConstraint: %v", p)
  1873  
  1874  	/*
  1875  		table_constraint:
  1876  			[ CONSTRAINT constraint_name ]
  1877  			{ check | foreign_key }
  1878  	*/
  1879  
  1880  	if p.eat("CONSTRAINT") {
  1881  		pos := p.Pos()
  1882  		// Named constraint.
  1883  		cname, err := p.parseTableOrIndexOrColumnName()
  1884  		if err != nil {
  1885  			return TableConstraint{}, err
  1886  		}
  1887  		c, err := p.parseConstraint()
  1888  		if err != nil {
  1889  			return TableConstraint{}, err
  1890  		}
  1891  		return TableConstraint{
  1892  			Name:       cname,
  1893  			Constraint: c,
  1894  			Position:   pos,
  1895  		}, nil
  1896  	}
  1897  
  1898  	// Unnamed constraint.
  1899  	c, err := p.parseConstraint()
  1900  	if err != nil {
  1901  		return TableConstraint{}, err
  1902  	}
  1903  	return TableConstraint{
  1904  		Constraint: c,
  1905  		Position:   c.Pos(),
  1906  	}, nil
  1907  }
  1908  
  1909  func (p *parser) parseConstraint() (Constraint, *parseError) {
  1910  	if p.sniff("FOREIGN") {
  1911  		fk, err := p.parseForeignKey()
  1912  		return fk, err
  1913  	}
  1914  	c, err := p.parseCheck()
  1915  	return c, err
  1916  }
  1917  
  1918  func (p *parser) parseForeignKey() (ForeignKey, *parseError) {
  1919  	debugf("parseForeignKey: %v", p)
  1920  
  1921  	/*
  1922  		foreign_key:
  1923  			FOREIGN KEY ( column_name [, ... ] ) REFERENCES ref_table ( ref_column [, ... ] )
  1924  	*/
  1925  
  1926  	if err := p.expect("FOREIGN"); err != nil {
  1927  		return ForeignKey{}, err
  1928  	}
  1929  	fk := ForeignKey{Position: p.Pos()}
  1930  	if err := p.expect("KEY"); err != nil {
  1931  		return ForeignKey{}, err
  1932  	}
  1933  	var err *parseError
  1934  	fk.Columns, err = p.parseColumnNameList()
  1935  	if err != nil {
  1936  		return ForeignKey{}, err
  1937  	}
  1938  	if err := p.expect("REFERENCES"); err != nil {
  1939  		return ForeignKey{}, err
  1940  	}
  1941  	fk.RefTable, err = p.parseTableOrIndexOrColumnName()
  1942  	if err != nil {
  1943  		return ForeignKey{}, err
  1944  	}
  1945  	fk.RefColumns, err = p.parseColumnNameList()
  1946  	if err != nil {
  1947  		return ForeignKey{}, err
  1948  	}
  1949  	return fk, nil
  1950  }
  1951  
  1952  func (p *parser) parseCheck() (Check, *parseError) {
  1953  	debugf("parseCheck: %v", p)
  1954  
  1955  	/*
  1956  		check:
  1957  			CHECK ( expression )
  1958  	*/
  1959  
  1960  	if err := p.expect("CHECK"); err != nil {
  1961  		return Check{}, err
  1962  	}
  1963  	c := Check{Position: p.Pos()}
  1964  	if err := p.expect("("); err != nil {
  1965  		return Check{}, err
  1966  	}
  1967  	var err *parseError
  1968  	c.Expr, err = p.parseBoolExpr()
  1969  	if err != nil {
  1970  		return Check{}, err
  1971  	}
  1972  	if err := p.expect(")"); err != nil {
  1973  		return Check{}, err
  1974  	}
  1975  	return c, nil
  1976  }
  1977  
  1978  func (p *parser) parseColumnNameList() ([]ID, *parseError) {
  1979  	var list []ID
  1980  	err := p.parseCommaList("(", ")", func(p *parser) *parseError {
  1981  		n, err := p.parseTableOrIndexOrColumnName()
  1982  		if err != nil {
  1983  			return err
  1984  		}
  1985  		list = append(list, n)
  1986  		return nil
  1987  	})
  1988  	return list, err
  1989  }
  1990  
  1991  var baseTypes = map[string]TypeBase{
  1992  	"BOOL":      Bool,
  1993  	"INT64":     Int64,
  1994  	"FLOAT64":   Float64,
  1995  	"NUMERIC":   Numeric,
  1996  	"STRING":    String,
  1997  	"BYTES":     Bytes,
  1998  	"DATE":      Date,
  1999  	"TIMESTAMP": Timestamp,
  2000  	"JSON":      JSON,
  2001  }
  2002  
  2003  func (p *parser) parseBaseType() (Type, *parseError) {
  2004  	return p.parseBaseOrParameterizedType(false)
  2005  }
  2006  
  2007  func (p *parser) parseType() (Type, *parseError) {
  2008  	return p.parseBaseOrParameterizedType(true)
  2009  }
  2010  
  2011  var extractPartTypes = map[string]TypeBase{
  2012  	"DAY":   Int64,
  2013  	"MONTH": Int64,
  2014  	"YEAR":  Int64,
  2015  	"DATE":  Date,
  2016  }
  2017  
  2018  func (p *parser) parseExtractType() (Type, string, *parseError) {
  2019  	var t Type
  2020  	tok := p.next()
  2021  	if tok.err != nil {
  2022  		return Type{}, "", tok.err
  2023  	}
  2024  	base, ok := extractPartTypes[strings.ToUpper(tok.value)] // valid part types for EXTRACT is keyed by upper case strings.
  2025  	if !ok {
  2026  		return Type{}, "", p.errorf("got %q, want valid EXTRACT types", tok.value)
  2027  	}
  2028  	t.Base = base
  2029  	return t, strings.ToUpper(tok.value), nil
  2030  }
  2031  
  2032  func (p *parser) parseBaseOrParameterizedType(withParam bool) (Type, *parseError) {
  2033  	debugf("parseBaseOrParameterizedType: %v", p)
  2034  
  2035  	/*
  2036  		array_type:
  2037  			ARRAY< scalar_type >
  2038  
  2039  		scalar_type:
  2040  			{ BOOL | INT64 | FLOAT64 | NUMERIC | STRING( length ) | BYTES( length ) | DATE | TIMESTAMP | JSON }
  2041  		length:
  2042  			{ int64_value | MAX }
  2043  	*/
  2044  
  2045  	var t Type
  2046  
  2047  	tok := p.next()
  2048  	if tok.err != nil {
  2049  		return Type{}, tok.err
  2050  	}
  2051  	if tok.caseEqual("ARRAY") {
  2052  		t.Array = true
  2053  		if err := p.expect("<"); err != nil {
  2054  			return Type{}, err
  2055  		}
  2056  		tok = p.next()
  2057  		if tok.err != nil {
  2058  			return Type{}, tok.err
  2059  		}
  2060  	}
  2061  	base, ok := baseTypes[strings.ToUpper(tok.value)] // baseTypes is keyed by upper case strings.
  2062  	if !ok {
  2063  		return Type{}, p.errorf("got %q, want scalar type", tok.value)
  2064  	}
  2065  	t.Base = base
  2066  
  2067  	if withParam && (t.Base == String || t.Base == Bytes) {
  2068  		if err := p.expect("("); err != nil {
  2069  			return Type{}, err
  2070  		}
  2071  
  2072  		tok = p.next()
  2073  		if tok.err != nil {
  2074  			return Type{}, tok.err
  2075  		}
  2076  		if tok.caseEqual("MAX") {
  2077  			t.Len = MaxLen
  2078  		} else if tok.typ == int64Token {
  2079  			n, err := strconv.ParseInt(tok.value, tok.int64Base, 64)
  2080  			if err != nil {
  2081  				return Type{}, p.errorf("%v", err)
  2082  			}
  2083  			t.Len = n
  2084  		} else {
  2085  			return Type{}, p.errorf("got %q, want MAX or int64", tok.value)
  2086  		}
  2087  
  2088  		if err := p.expect(")"); err != nil {
  2089  			return Type{}, err
  2090  		}
  2091  	}
  2092  
  2093  	if t.Array {
  2094  		if err := p.expect(">"); err != nil {
  2095  			return Type{}, err
  2096  		}
  2097  	}
  2098  
  2099  	return t, nil
  2100  }
  2101  
  2102  func (p *parser) parseQuery() (Query, *parseError) {
  2103  	debugf("parseQuery: %v", p)
  2104  
  2105  	/*
  2106  		query_statement:
  2107  			[ table_hint_expr ][ join_hint_expr ]
  2108  			query_expr
  2109  
  2110  		query_expr:
  2111  			{ select | ( query_expr ) | query_expr set_op query_expr }
  2112  			[ ORDER BY expression [{ ASC | DESC }] [, ...] ]
  2113  			[ LIMIT count [ OFFSET skip_rows ] ]
  2114  	*/
  2115  
  2116  	// TODO: sub-selects, etc.
  2117  
  2118  	if err := p.expect("SELECT"); err != nil {
  2119  		return Query{}, err
  2120  	}
  2121  	p.back()
  2122  	sel, err := p.parseSelect()
  2123  	if err != nil {
  2124  		return Query{}, err
  2125  	}
  2126  	q := Query{Select: sel}
  2127  
  2128  	if p.eat("ORDER", "BY") {
  2129  		for {
  2130  			o, err := p.parseOrder()
  2131  			if err != nil {
  2132  				return Query{}, err
  2133  			}
  2134  			q.Order = append(q.Order, o)
  2135  
  2136  			if !p.eat(",") {
  2137  				break
  2138  			}
  2139  		}
  2140  	}
  2141  
  2142  	if p.eat("LIMIT") {
  2143  		// "only literal or parameter values"
  2144  		// https://cloud.google.com/spanner/docs/query-syntax#limit-clause-and-offset-clause
  2145  
  2146  		lim, err := p.parseLiteralOrParam()
  2147  		if err != nil {
  2148  			return Query{}, err
  2149  		}
  2150  		q.Limit = lim
  2151  
  2152  		if p.eat("OFFSET") {
  2153  			off, err := p.parseLiteralOrParam()
  2154  			if err != nil {
  2155  				return Query{}, err
  2156  			}
  2157  			q.Offset = off
  2158  		}
  2159  	}
  2160  
  2161  	return q, nil
  2162  }
  2163  
  2164  func (p *parser) parseSelect() (Select, *parseError) {
  2165  	debugf("parseSelect: %v", p)
  2166  
  2167  	/*
  2168  		select:
  2169  			SELECT  [{ ALL | DISTINCT }]
  2170  				{ [ expression. ]* | expression [ [ AS ] alias ] } [, ...]
  2171  			[ FROM from_item [ tablesample_type ] [, ...] ]
  2172  			[ WHERE bool_expression ]
  2173  			[ GROUP BY expression [, ...] ]
  2174  			[ HAVING bool_expression ]
  2175  	*/
  2176  	if err := p.expect("SELECT"); err != nil {
  2177  		return Select{}, err
  2178  	}
  2179  
  2180  	var sel Select
  2181  
  2182  	if p.eat("ALL") {
  2183  		// Nothing to do; this is the default.
  2184  	} else if p.eat("DISTINCT") {
  2185  		sel.Distinct = true
  2186  	}
  2187  
  2188  	// Read expressions for the SELECT list.
  2189  	list, aliases, err := p.parseSelectList()
  2190  	if err != nil {
  2191  		return Select{}, err
  2192  	}
  2193  	sel.List, sel.ListAliases = list, aliases
  2194  
  2195  	if p.eat("FROM") {
  2196  		padTS := func() {
  2197  			for len(sel.TableSamples) < len(sel.From) {
  2198  				sel.TableSamples = append(sel.TableSamples, nil)
  2199  			}
  2200  		}
  2201  
  2202  		for {
  2203  			from, err := p.parseSelectFrom()
  2204  			if err != nil {
  2205  				return Select{}, err
  2206  			}
  2207  			sel.From = append(sel.From, from)
  2208  
  2209  			if p.sniff("TABLESAMPLE") {
  2210  				ts, err := p.parseTableSample()
  2211  				if err != nil {
  2212  					return Select{}, err
  2213  				}
  2214  				padTS()
  2215  				sel.TableSamples[len(sel.TableSamples)-1] = &ts
  2216  			}
  2217  
  2218  			if p.eat(",") {
  2219  				continue
  2220  			}
  2221  			break
  2222  		}
  2223  
  2224  		if sel.TableSamples != nil {
  2225  			padTS()
  2226  		}
  2227  	}
  2228  
  2229  	if p.eat("WHERE") {
  2230  		where, err := p.parseBoolExpr()
  2231  		if err != nil {
  2232  			return Select{}, err
  2233  		}
  2234  		sel.Where = where
  2235  	}
  2236  
  2237  	if p.eat("GROUP", "BY") {
  2238  		list, err := p.parseExprList()
  2239  		if err != nil {
  2240  			return Select{}, err
  2241  		}
  2242  		sel.GroupBy = list
  2243  	}
  2244  
  2245  	// TODO: HAVING
  2246  
  2247  	return sel, nil
  2248  }
  2249  
  2250  func (p *parser) parseSelectList() ([]Expr, []ID, *parseError) {
  2251  	var list []Expr
  2252  	var aliases []ID // Only set if any aliases are seen.
  2253  	padAliases := func() {
  2254  		for len(aliases) < len(list) {
  2255  			aliases = append(aliases, "")
  2256  		}
  2257  	}
  2258  
  2259  	for {
  2260  		expr, err := p.parseExpr()
  2261  		if err != nil {
  2262  			return nil, nil, err
  2263  		}
  2264  		list = append(list, expr)
  2265  
  2266  		// TODO: The "AS" keyword is optional.
  2267  		if p.eat("AS") {
  2268  			alias, err := p.parseAlias()
  2269  			if err != nil {
  2270  				return nil, nil, err
  2271  			}
  2272  
  2273  			padAliases()
  2274  			aliases[len(aliases)-1] = alias
  2275  		}
  2276  
  2277  		if p.eat(",") {
  2278  			continue
  2279  		}
  2280  		break
  2281  	}
  2282  	if aliases != nil {
  2283  		padAliases()
  2284  	}
  2285  	return list, aliases, nil
  2286  }
  2287  
  2288  func (p *parser) parseSelectFromTable() (SelectFrom, *parseError) {
  2289  	if p.eat("UNNEST") {
  2290  		if err := p.expect("("); err != nil {
  2291  			return nil, err
  2292  		}
  2293  		e, err := p.parseExpr()
  2294  		if err != nil {
  2295  			return nil, err
  2296  		}
  2297  		if err := p.expect(")"); err != nil {
  2298  			return nil, err
  2299  		}
  2300  		sfu := SelectFromUnnest{Expr: e}
  2301  		if p.eat("AS") { // TODO: The "AS" keyword is optional.
  2302  			alias, err := p.parseAlias()
  2303  			if err != nil {
  2304  				return nil, err
  2305  			}
  2306  			sfu.Alias = alias
  2307  		}
  2308  		// TODO: hint, offset
  2309  		return sfu, nil
  2310  	}
  2311  
  2312  	// A join starts with a from_item, so that can't be detected in advance.
  2313  	// TODO: Support subquery, field_path, array_path, WITH.
  2314  	// TODO: Verify associativity of multile joins.
  2315  
  2316  	tname, err := p.parseTableOrIndexOrColumnName()
  2317  	if err != nil {
  2318  		return nil, err
  2319  	}
  2320  	sf := SelectFromTable{Table: tname}
  2321  	if p.eat("@") {
  2322  		hints, err := p.parseHints(map[string]string{})
  2323  		if err != nil {
  2324  			return nil, err
  2325  		}
  2326  		sf.Hints = hints
  2327  	}
  2328  
  2329  	// TODO: The "AS" keyword is optional.
  2330  	if p.eat("AS") {
  2331  		alias, err := p.parseAlias()
  2332  		if err != nil {
  2333  			return nil, err
  2334  		}
  2335  		sf.Alias = alias
  2336  	}
  2337  	return sf, nil
  2338  }
  2339  
  2340  func (p *parser) parseSelectFromJoin(lhs SelectFrom) (SelectFrom, *parseError) {
  2341  	// Look ahead to see if this is a join.
  2342  	tok := p.next()
  2343  	if tok.err != nil {
  2344  		p.back()
  2345  		return nil, nil
  2346  	}
  2347  	var hashJoin bool // Special case for "HASH JOIN" syntax.
  2348  	if tok.caseEqual("HASH") {
  2349  		hashJoin = true
  2350  		tok = p.next()
  2351  		if tok.err != nil {
  2352  			return nil, tok.err
  2353  		}
  2354  	}
  2355  	var jt JoinType
  2356  	if tok.caseEqual("JOIN") {
  2357  		// This is implicitly an inner join.
  2358  		jt = InnerJoin
  2359  	} else if j, ok := joinKeywords[tok.value]; ok {
  2360  		jt = j
  2361  		switch jt {
  2362  		case FullJoin, LeftJoin, RightJoin:
  2363  			// These join types are implicitly "outer" joins,
  2364  			// so the "OUTER" keyword is optional.
  2365  			p.eat("OUTER")
  2366  		}
  2367  		if err := p.expect("JOIN"); err != nil {
  2368  			return nil, err
  2369  		}
  2370  	} else {
  2371  		// Not a join
  2372  		p.back()
  2373  		return nil, nil
  2374  	}
  2375  	sfj := SelectFromJoin{
  2376  		Type: jt,
  2377  		LHS:  lhs,
  2378  	}
  2379  	var hints map[string]string
  2380  	if hashJoin {
  2381  		hints = map[string]string{}
  2382  		hints["JOIN_METHOD"] = "HASH_JOIN"
  2383  	}
  2384  
  2385  	if p.eat("@") {
  2386  		h, err := p.parseHints(hints)
  2387  		if err != nil {
  2388  			return nil, err
  2389  		}
  2390  		hints = h
  2391  	}
  2392  	sfj.Hints = hints
  2393  
  2394  	rhs, err := p.parseSelectFromTable()
  2395  	if err != nil {
  2396  		return nil, err
  2397  	}
  2398  
  2399  	sfj.RHS = rhs
  2400  
  2401  	if p.eat("ON") {
  2402  		sfj.On, err = p.parseBoolExpr()
  2403  		if err != nil {
  2404  			return nil, err
  2405  		}
  2406  	}
  2407  	if p.eat("USING") {
  2408  		if sfj.On != nil {
  2409  			return nil, p.errorf("join may not have both ON and USING clauses")
  2410  		}
  2411  		sfj.Using, err = p.parseColumnNameList()
  2412  		if err != nil {
  2413  			return nil, err
  2414  		}
  2415  	}
  2416  
  2417  	return sfj, nil
  2418  }
  2419  
  2420  func (p *parser) parseSelectFrom() (SelectFrom, *parseError) {
  2421  	debugf("parseSelectFrom: %v", p)
  2422  
  2423  	/*
  2424  		from_item: {
  2425  			table_name [ table_hint_expr ] [ [ AS ] alias ] |
  2426  			join |
  2427  			( query_expr ) [ table_hint_expr ] [ [ AS ] alias ] |
  2428  			field_path |
  2429  			{ UNNEST( array_expression ) | UNNEST( array_path ) | array_path }
  2430  				[ table_hint_expr ] [ [ AS ] alias ] [ WITH OFFSET [ [ AS ] alias ] ] |
  2431  			with_query_name [ table_hint_expr ] [ [ AS ] alias ]
  2432  		}
  2433  
  2434  		join:
  2435  			from_item [ join_type ] [ join_method ] JOIN  [ join_hint_expr ] from_item
  2436  				[ ON bool_expression | USING ( join_column [, ...] ) ]
  2437  
  2438  		join_type:
  2439  			{ INNER | CROSS | FULL [OUTER] | LEFT [OUTER] | RIGHT [OUTER] }
  2440  	*/
  2441  	leftHandSide, err := p.parseSelectFromTable()
  2442  	if err != nil {
  2443  		return nil, err
  2444  	}
  2445  	// Lets keep consuming joins until we no longer find more joins
  2446  	for {
  2447  		sfj, err := p.parseSelectFromJoin(leftHandSide)
  2448  		if err != nil {
  2449  			return nil, err
  2450  		}
  2451  		if sfj == nil {
  2452  			// There was no join to consume
  2453  			break
  2454  		}
  2455  		leftHandSide = sfj
  2456  	}
  2457  	return leftHandSide, nil
  2458  }
  2459  
  2460  var joinKeywords = map[string]JoinType{
  2461  	"INNER": InnerJoin,
  2462  	"CROSS": CrossJoin,
  2463  	"FULL":  FullJoin,
  2464  	"LEFT":  LeftJoin,
  2465  	"RIGHT": RightJoin,
  2466  }
  2467  
  2468  func (p *parser) parseTableSample() (TableSample, *parseError) {
  2469  	var ts TableSample
  2470  
  2471  	if err := p.expect("TABLESAMPLE"); err != nil {
  2472  		return ts, err
  2473  	}
  2474  
  2475  	tok := p.next()
  2476  	switch {
  2477  	case tok.err != nil:
  2478  		return ts, tok.err
  2479  	case tok.caseEqual("BERNOULLI"):
  2480  		ts.Method = Bernoulli
  2481  	case tok.caseEqual("RESERVOIR"):
  2482  		ts.Method = Reservoir
  2483  	default:
  2484  		return ts, p.errorf("got %q, want BERNOULLI or RESERVOIR", tok.value)
  2485  	}
  2486  
  2487  	if err := p.expect("("); err != nil {
  2488  		return ts, err
  2489  	}
  2490  
  2491  	// The docs say "numeric_value_expression" here,
  2492  	// but that doesn't appear to be defined anywhere.
  2493  	size, err := p.parseExpr()
  2494  	if err != nil {
  2495  		return ts, err
  2496  	}
  2497  	ts.Size = size
  2498  
  2499  	tok = p.next()
  2500  	switch {
  2501  	case tok.err != nil:
  2502  		return ts, tok.err
  2503  	case tok.caseEqual("PERCENT"):
  2504  		ts.SizeType = PercentTableSample
  2505  	case tok.caseEqual("ROWS"):
  2506  		ts.SizeType = RowsTableSample
  2507  	default:
  2508  		return ts, p.errorf("got %q, want PERCENT or ROWS", tok.value)
  2509  	}
  2510  
  2511  	if err := p.expect(")"); err != nil {
  2512  		return ts, err
  2513  	}
  2514  
  2515  	return ts, nil
  2516  }
  2517  
  2518  func (p *parser) parseOrder() (Order, *parseError) {
  2519  	/*
  2520  		expression [{ ASC | DESC }]
  2521  	*/
  2522  
  2523  	expr, err := p.parseExpr()
  2524  	if err != nil {
  2525  		return Order{}, err
  2526  	}
  2527  	o := Order{Expr: expr}
  2528  
  2529  	if p.eat("ASC") {
  2530  		// OK.
  2531  	} else if p.eat("DESC") {
  2532  		o.Desc = true
  2533  	}
  2534  
  2535  	return o, nil
  2536  }
  2537  
  2538  func (p *parser) parseLiteralOrParam() (LiteralOrParam, *parseError) {
  2539  	tok := p.next()
  2540  	if tok.err != nil {
  2541  		return nil, tok.err
  2542  	}
  2543  	if tok.typ == int64Token {
  2544  		n, err := strconv.ParseInt(tok.value, tok.int64Base, 64)
  2545  		if err != nil {
  2546  			return nil, p.errorf("%v", err)
  2547  		}
  2548  		return IntegerLiteral(n), nil
  2549  	}
  2550  	// TODO: check character sets.
  2551  	if strings.HasPrefix(tok.value, "@") {
  2552  		return Param(tok.value[1:]), nil
  2553  	}
  2554  	return nil, p.errorf("got %q, want literal or parameter", tok.value)
  2555  }
  2556  
  2557  func (p *parser) parseExprList() ([]Expr, *parseError) {
  2558  	var list []Expr
  2559  	for {
  2560  		expr, err := p.parseExpr()
  2561  		if err != nil {
  2562  			return nil, err
  2563  		}
  2564  		list = append(list, expr)
  2565  
  2566  		if p.eat(",") {
  2567  			continue
  2568  		}
  2569  		break
  2570  	}
  2571  	return list, nil
  2572  }
  2573  
  2574  func (p *parser) parseParenExprList() ([]Expr, *parseError) {
  2575  	return p.parseParenExprListWithParseFunc(func(p *parser) (Expr, *parseError) {
  2576  		return p.parseExpr()
  2577  	})
  2578  }
  2579  
  2580  func (p *parser) parseParenExprListWithParseFunc(f func(*parser) (Expr, *parseError)) ([]Expr, *parseError) {
  2581  	var list []Expr
  2582  	err := p.parseCommaList("(", ")", func(p *parser) *parseError {
  2583  		e, err := f(p)
  2584  		if err != nil {
  2585  			return err
  2586  		}
  2587  		list = append(list, e)
  2588  		return nil
  2589  	})
  2590  	return list, err
  2591  }
  2592  
  2593  // Special argument parser for CAST and SAFE_CAST
  2594  var typedArgParser = func(p *parser) (Expr, *parseError) {
  2595  	e, err := p.parseExpr()
  2596  	if err != nil {
  2597  		return nil, err
  2598  	}
  2599  	if err := p.expect("AS"); err != nil {
  2600  		return nil, err
  2601  	}
  2602  	// typename in cast function must not be parameterized types
  2603  	toType, err := p.parseBaseType()
  2604  	if err != nil {
  2605  		return nil, err
  2606  	}
  2607  	return TypedExpr{
  2608  		Expr: e,
  2609  		Type: toType,
  2610  	}, nil
  2611  }
  2612  
  2613  // Special argument parser for EXTRACT
  2614  var extractArgParser = func(p *parser) (Expr, *parseError) {
  2615  	partType, part, err := p.parseExtractType()
  2616  	if err != nil {
  2617  		return nil, err
  2618  	}
  2619  	if err := p.expect("FROM"); err != nil {
  2620  		return nil, err
  2621  	}
  2622  	e, err := p.parseExpr()
  2623  	if err != nil {
  2624  		return nil, err
  2625  	}
  2626  	// AT TIME ZONE is optional
  2627  	if p.eat("AT", "TIME", "ZONE") {
  2628  		tok := p.next()
  2629  		if tok.err != nil {
  2630  			return nil, err
  2631  		}
  2632  		return ExtractExpr{Part: part, Type: partType, Expr: AtTimeZoneExpr{Expr: e, Zone: tok.string, Type: Type{Base: Timestamp}}}, nil
  2633  	}
  2634  	return ExtractExpr{
  2635  		Part: part,
  2636  		Expr: e,
  2637  		Type: partType,
  2638  	}, nil
  2639  }
  2640  
  2641  /*
  2642  Expressions
  2643  
  2644  Cloud Spanner expressions are not formally specified.
  2645  The set of operators and their precedence is listed in
  2646  https://cloud.google.com/spanner/docs/functions-and-operators#operators.
  2647  
  2648  parseExpr works as a classical recursive descent parser, splitting
  2649  precedence levels into separate methods, where the call stack is in
  2650  ascending order of precedence:
  2651  	parseExpr
  2652  	orParser
  2653  	andParser
  2654  	parseIsOp
  2655  	parseInOp
  2656  	parseComparisonOp
  2657  	parseArithOp: |, ^, &, << and >>, + and -, * and / and ||
  2658  	parseUnaryArithOp: - and ~
  2659  	parseLit
  2660  */
  2661  
  2662  func (p *parser) parseExpr() (Expr, *parseError) {
  2663  	debugf("parseExpr: %v", p)
  2664  
  2665  	return orParser.parse(p)
  2666  }
  2667  
  2668  // binOpParser is a generic meta-parser for binary operations.
  2669  // It assumes the operation is left associative.
  2670  type binOpParser struct {
  2671  	LHS, RHS func(*parser) (Expr, *parseError)
  2672  	Op       string
  2673  	ArgCheck func(Expr) error
  2674  	Combiner func(lhs, rhs Expr) Expr
  2675  }
  2676  
  2677  func (bin binOpParser) parse(p *parser) (Expr, *parseError) {
  2678  	expr, err := bin.LHS(p)
  2679  	if err != nil {
  2680  		return nil, err
  2681  	}
  2682  
  2683  	for {
  2684  		if !p.eat(bin.Op) {
  2685  			break
  2686  		}
  2687  		rhs, err := bin.RHS(p)
  2688  		if err != nil {
  2689  			return nil, err
  2690  		}
  2691  		if bin.ArgCheck != nil {
  2692  			if err := bin.ArgCheck(expr); err != nil {
  2693  				return nil, p.errorf("%v", err)
  2694  			}
  2695  			if err := bin.ArgCheck(rhs); err != nil {
  2696  				return nil, p.errorf("%v", err)
  2697  			}
  2698  		}
  2699  		expr = bin.Combiner(expr, rhs)
  2700  	}
  2701  	return expr, nil
  2702  }
  2703  
  2704  // Break initialisation loop.
  2705  func init() { orParser = orParserShim }
  2706  
  2707  var (
  2708  	boolExprCheck = func(expr Expr) error {
  2709  		if _, ok := expr.(BoolExpr); !ok {
  2710  			return fmt.Errorf("got %T, want a boolean expression", expr)
  2711  		}
  2712  		return nil
  2713  	}
  2714  
  2715  	orParser binOpParser
  2716  
  2717  	orParserShim = binOpParser{
  2718  		LHS:      andParser.parse,
  2719  		RHS:      andParser.parse,
  2720  		Op:       "OR",
  2721  		ArgCheck: boolExprCheck,
  2722  		Combiner: func(lhs, rhs Expr) Expr {
  2723  			return LogicalOp{LHS: lhs.(BoolExpr), Op: Or, RHS: rhs.(BoolExpr)}
  2724  		},
  2725  	}
  2726  	andParser = binOpParser{
  2727  		LHS:      (*parser).parseLogicalNot,
  2728  		RHS:      (*parser).parseLogicalNot,
  2729  		Op:       "AND",
  2730  		ArgCheck: boolExprCheck,
  2731  		Combiner: func(lhs, rhs Expr) Expr {
  2732  			return LogicalOp{LHS: lhs.(BoolExpr), Op: And, RHS: rhs.(BoolExpr)}
  2733  		},
  2734  	}
  2735  
  2736  	bitOrParser  = newBinArithParser("|", BitOr, bitXorParser.parse)
  2737  	bitXorParser = newBinArithParser("^", BitXor, bitAndParser.parse)
  2738  	bitAndParser = newBinArithParser("&", BitAnd, bitShrParser.parse)
  2739  	bitShrParser = newBinArithParser(">>", BitShr, bitShlParser.parse)
  2740  	bitShlParser = newBinArithParser("<<", BitShl, subParser.parse)
  2741  	subParser    = newBinArithParser("-", Sub, addParser.parse)
  2742  	addParser    = newBinArithParser("+", Add, concatParser.parse)
  2743  	concatParser = newBinArithParser("||", Concat, divParser.parse)
  2744  	divParser    = newBinArithParser("/", Div, mulParser.parse)
  2745  	mulParser    = newBinArithParser("*", Mul, (*parser).parseUnaryArithOp)
  2746  )
  2747  
  2748  func newBinArithParser(opStr string, op ArithOperator, nextPrec func(*parser) (Expr, *parseError)) binOpParser {
  2749  	return binOpParser{
  2750  		LHS: nextPrec,
  2751  		RHS: nextPrec,
  2752  		Op:  opStr,
  2753  		// TODO: ArgCheck? numeric inputs only, except for ||.
  2754  		Combiner: func(lhs, rhs Expr) Expr {
  2755  			return ArithOp{LHS: lhs, Op: op, RHS: rhs}
  2756  		},
  2757  	}
  2758  }
  2759  
  2760  func (p *parser) parseLogicalNot() (Expr, *parseError) {
  2761  	if !p.eat("NOT") {
  2762  		return p.parseIsOp()
  2763  	}
  2764  	be, err := p.parseBoolExpr()
  2765  	if err != nil {
  2766  		return nil, err
  2767  	}
  2768  	return LogicalOp{Op: Not, RHS: be}, nil
  2769  }
  2770  
  2771  func (p *parser) parseIsOp() (Expr, *parseError) {
  2772  	debugf("parseIsOp: %v", p)
  2773  
  2774  	expr, err := p.parseInOp()
  2775  	if err != nil {
  2776  		return nil, err
  2777  	}
  2778  
  2779  	if !p.eat("IS") {
  2780  		return expr, nil
  2781  	}
  2782  
  2783  	isOp := IsOp{LHS: expr}
  2784  	if p.eat("NOT") {
  2785  		isOp.Neg = true
  2786  	}
  2787  
  2788  	tok := p.next()
  2789  	if tok.err != nil {
  2790  		return nil, tok.err
  2791  	}
  2792  	switch {
  2793  	case tok.caseEqual("NULL"):
  2794  		isOp.RHS = Null
  2795  	case tok.caseEqual("TRUE"):
  2796  		isOp.RHS = True
  2797  	case tok.caseEqual("FALSE"):
  2798  		isOp.RHS = False
  2799  	default:
  2800  		return nil, p.errorf("got %q, want NULL or TRUE or FALSE", tok.value)
  2801  	}
  2802  
  2803  	return isOp, nil
  2804  }
  2805  
  2806  func (p *parser) parseInOp() (Expr, *parseError) {
  2807  	debugf("parseInOp: %v", p)
  2808  
  2809  	expr, err := p.parseComparisonOp()
  2810  	if err != nil {
  2811  		return nil, err
  2812  	}
  2813  
  2814  	inOp := InOp{LHS: expr}
  2815  	if p.eat("NOT", "IN") {
  2816  		inOp.Neg = true
  2817  	} else if p.eat("IN") {
  2818  		// Okay.
  2819  	} else {
  2820  		return expr, nil
  2821  	}
  2822  
  2823  	if p.eat("UNNEST") {
  2824  		inOp.Unnest = true
  2825  	}
  2826  
  2827  	inOp.RHS, err = p.parseParenExprList()
  2828  	if err != nil {
  2829  		return nil, err
  2830  	}
  2831  	return inOp, nil
  2832  }
  2833  
  2834  var symbolicOperators = map[string]ComparisonOperator{
  2835  	"<":  Lt,
  2836  	"<=": Le,
  2837  	">":  Gt,
  2838  	">=": Ge,
  2839  	"=":  Eq,
  2840  	"!=": Ne,
  2841  	"<>": Ne,
  2842  }
  2843  
  2844  func (p *parser) parseComparisonOp() (Expr, *parseError) {
  2845  	debugf("parseComparisonOp: %v", p)
  2846  
  2847  	expr, err := p.parseArithOp()
  2848  	if err != nil {
  2849  		return nil, err
  2850  	}
  2851  
  2852  	for {
  2853  		// There's a need for two token lookahead.
  2854  		var op ComparisonOperator
  2855  		var rhs2 bool
  2856  		if p.eat("NOT", "LIKE") {
  2857  			op = NotLike
  2858  		} else if p.eat("NOT", "BETWEEN") {
  2859  			op, rhs2 = NotBetween, true
  2860  		} else if p.eat("LIKE") {
  2861  			op = Like
  2862  		} else if p.eat("BETWEEN") {
  2863  			op, rhs2 = Between, true
  2864  		} else {
  2865  			// Check for a symbolic operator.
  2866  			tok := p.next()
  2867  			if tok.err != nil {
  2868  				p.back()
  2869  				break
  2870  			}
  2871  			var ok bool
  2872  			op, ok = symbolicOperators[tok.value]
  2873  			if !ok {
  2874  				p.back()
  2875  				break
  2876  			}
  2877  		}
  2878  
  2879  		rhs, err := p.parseArithOp()
  2880  		if err != nil {
  2881  			return nil, err
  2882  		}
  2883  		co := ComparisonOp{LHS: expr, Op: op, RHS: rhs}
  2884  
  2885  		if rhs2 {
  2886  			if err := p.expect("AND"); err != nil {
  2887  				return nil, err
  2888  			}
  2889  			rhs2, err := p.parseArithOp()
  2890  			if err != nil {
  2891  				return nil, err
  2892  			}
  2893  			co.RHS2 = rhs2
  2894  		}
  2895  
  2896  		expr = co
  2897  	}
  2898  	return expr, nil
  2899  }
  2900  
  2901  func (p *parser) parseArithOp() (Expr, *parseError) {
  2902  	return bitOrParser.parse(p)
  2903  }
  2904  
  2905  var unaryArithOperators = map[string]ArithOperator{
  2906  	"-": Neg,
  2907  	"~": BitNot,
  2908  	"+": Plus,
  2909  }
  2910  
  2911  func (p *parser) parseUnaryArithOp() (Expr, *parseError) {
  2912  	tok := p.next()
  2913  	if tok.err != nil {
  2914  		return nil, tok.err
  2915  	}
  2916  
  2917  	op := tok.value
  2918  
  2919  	if op == "-" || op == "+" {
  2920  		// If the next token is a numeric token, combine and parse as a literal.
  2921  		ntok := p.next()
  2922  		if ntok.err == nil {
  2923  			switch ntok.typ {
  2924  			case int64Token:
  2925  				comb := op + ntok.value
  2926  				n, err := strconv.ParseInt(comb, ntok.int64Base, 64)
  2927  				if err != nil {
  2928  					return nil, p.errorf("%v", err)
  2929  				}
  2930  				return IntegerLiteral(n), nil
  2931  			case float64Token:
  2932  				f := ntok.float64
  2933  				if op == "-" {
  2934  					f = -f
  2935  				}
  2936  				return FloatLiteral(f), nil
  2937  			}
  2938  		}
  2939  		// It is not possible for the p.back() lower down to fire
  2940  		// because - and + are in unaryArithOperators.
  2941  		p.back()
  2942  	}
  2943  
  2944  	if op, ok := unaryArithOperators[op]; ok {
  2945  		e, err := p.parseLit()
  2946  		if err != nil {
  2947  			return nil, err
  2948  		}
  2949  		return ArithOp{Op: op, RHS: e}, nil
  2950  	}
  2951  	p.back()
  2952  
  2953  	return p.parseLit()
  2954  }
  2955  
  2956  func (p *parser) parseLit() (Expr, *parseError) {
  2957  	tok := p.next()
  2958  	if tok.err != nil {
  2959  		return nil, tok.err
  2960  	}
  2961  
  2962  	switch tok.typ {
  2963  	case int64Token:
  2964  		n, err := strconv.ParseInt(tok.value, tok.int64Base, 64)
  2965  		if err != nil {
  2966  			return nil, p.errorf("%v", err)
  2967  		}
  2968  		return IntegerLiteral(n), nil
  2969  	case float64Token:
  2970  		return FloatLiteral(tok.float64), nil
  2971  	case stringToken:
  2972  		return StringLiteral(tok.string), nil
  2973  	case bytesToken:
  2974  		return BytesLiteral(tok.string), nil
  2975  	}
  2976  
  2977  	// Handle parenthesized expressions.
  2978  	if tok.value == "(" {
  2979  		e, err := p.parseExpr()
  2980  		if err != nil {
  2981  			return nil, err
  2982  		}
  2983  		if err := p.expect(")"); err != nil {
  2984  			return nil, err
  2985  		}
  2986  		return Paren{Expr: e}, nil
  2987  	}
  2988  
  2989  	// If the literal was an identifier, and there's an open paren next,
  2990  	// this is a function invocation.
  2991  	// The `funcs` map is keyed by upper case strings.
  2992  	if name := strings.ToUpper(tok.value); funcs[name] && p.sniff("(") {
  2993  		var list []Expr
  2994  		var err *parseError
  2995  		if f, ok := funcArgParsers[name]; ok {
  2996  			list, err = p.parseParenExprListWithParseFunc(f)
  2997  		} else {
  2998  			list, err = p.parseParenExprList()
  2999  		}
  3000  		if err != nil {
  3001  			return nil, err
  3002  		}
  3003  		return Func{
  3004  			Name: name,
  3005  			Args: list,
  3006  		}, nil
  3007  	}
  3008  
  3009  	// Handle some reserved keywords and special tokens that become specific values.
  3010  	switch {
  3011  	case tok.caseEqual("TRUE"):
  3012  		return True, nil
  3013  	case tok.caseEqual("FALSE"):
  3014  		return False, nil
  3015  	case tok.caseEqual("NULL"):
  3016  		return Null, nil
  3017  	case tok.value == "*":
  3018  		return Star, nil
  3019  	default:
  3020  		// TODO: Check IsKeyWord(tok.value), and return a good error?
  3021  	}
  3022  
  3023  	// Handle conditional expressions.
  3024  	switch {
  3025  	case tok.caseEqual("CASE"):
  3026  		p.back()
  3027  		return p.parseCaseExpr()
  3028  	}
  3029  
  3030  	// Handle typed literals.
  3031  	switch {
  3032  	case tok.caseEqual("ARRAY") || tok.value == "[":
  3033  		p.back()
  3034  		return p.parseArrayLit()
  3035  	case tok.caseEqual("DATE"):
  3036  		if p.sniffTokenType(stringToken) {
  3037  			p.back()
  3038  			return p.parseDateLit()
  3039  		}
  3040  	case tok.caseEqual("TIMESTAMP"):
  3041  		if p.sniffTokenType(stringToken) {
  3042  			p.back()
  3043  			return p.parseTimestampLit()
  3044  		}
  3045  	case tok.caseEqual("JSON"):
  3046  		if p.sniffTokenType(stringToken) {
  3047  			p.back()
  3048  			return p.parseJSONLit()
  3049  		}
  3050  	}
  3051  
  3052  	// TODO: struct literals
  3053  
  3054  	// Try a parameter.
  3055  	// TODO: check character sets.
  3056  	if strings.HasPrefix(tok.value, "@") {
  3057  		return Param(tok.value[1:]), nil
  3058  	}
  3059  
  3060  	// Only thing left is a path expression or standalone identifier.
  3061  	p.back()
  3062  	pe, err := p.parsePathExp()
  3063  	if err != nil {
  3064  		return nil, err
  3065  	}
  3066  	if len(pe) == 1 {
  3067  		return pe[0], nil // identifier
  3068  	}
  3069  	return pe, nil
  3070  }
  3071  
  3072  func (p *parser) parseCaseExpr() (Case, *parseError) {
  3073  	if err := p.expect("CASE"); err != nil {
  3074  		return Case{}, err
  3075  	}
  3076  
  3077  	var expr Expr
  3078  	if !p.sniff("WHEN") {
  3079  		var err *parseError
  3080  		expr, err = p.parseExpr()
  3081  		if err != nil {
  3082  			return Case{}, err
  3083  		}
  3084  	}
  3085  
  3086  	when, err := p.parseWhenClause()
  3087  	if err != nil {
  3088  		return Case{}, err
  3089  	}
  3090  	whens := []WhenClause{when}
  3091  	for p.sniff("WHEN") {
  3092  		when, err := p.parseWhenClause()
  3093  		if err != nil {
  3094  			return Case{}, err
  3095  		}
  3096  		whens = append(whens, when)
  3097  	}
  3098  
  3099  	var elseResult Expr
  3100  	if p.sniff("ELSE") {
  3101  		p.eat("ELSE")
  3102  		var err *parseError
  3103  		elseResult, err = p.parseExpr()
  3104  		if err != nil {
  3105  			return Case{}, err
  3106  		}
  3107  	}
  3108  
  3109  	if err := p.expect("END"); err != nil {
  3110  		return Case{}, err
  3111  	}
  3112  
  3113  	return Case{
  3114  		Expr:        expr,
  3115  		WhenClauses: whens,
  3116  		ElseResult:  elseResult,
  3117  	}, nil
  3118  }
  3119  
  3120  func (p *parser) parseWhenClause() (WhenClause, *parseError) {
  3121  	if err := p.expect("WHEN"); err != nil {
  3122  		return WhenClause{}, err
  3123  	}
  3124  	cond, err := p.parseExpr()
  3125  	if err != nil {
  3126  		return WhenClause{}, err
  3127  	}
  3128  	if err := p.expect("THEN"); err != nil {
  3129  		return WhenClause{}, err
  3130  	}
  3131  	result, err := p.parseExpr()
  3132  	if err != nil {
  3133  		return WhenClause{}, err
  3134  	}
  3135  	return WhenClause{Cond: cond, Result: result}, nil
  3136  }
  3137  
  3138  func (p *parser) parseArrayLit() (Array, *parseError) {
  3139  	// ARRAY keyword is optional.
  3140  	// TODO: If it is present, consume any <T> after it.
  3141  	p.eat("ARRAY")
  3142  
  3143  	var arr Array
  3144  	err := p.parseCommaList("[", "]", func(p *parser) *parseError {
  3145  		e, err := p.parseLit()
  3146  		if err != nil {
  3147  			return err
  3148  		}
  3149  		// TODO: Do type consistency checking here?
  3150  		arr = append(arr, e)
  3151  		return nil
  3152  	})
  3153  	return arr, err
  3154  }
  3155  
  3156  // TODO: There should be exported Parse{Date,Timestamp}Literal package-level funcs
  3157  // to support spannertest coercing plain string literals when used in a typed context.
  3158  // Those should wrap parseDateLit and parseTimestampLit below.
  3159  
  3160  func (p *parser) parseDateLit() (DateLiteral, *parseError) {
  3161  	if err := p.expect("DATE"); err != nil {
  3162  		return DateLiteral{}, err
  3163  	}
  3164  	s, err := p.parseStringLit()
  3165  	if err != nil {
  3166  		return DateLiteral{}, err
  3167  	}
  3168  	d, perr := civil.ParseDate(string(s))
  3169  	if perr != nil {
  3170  		return DateLiteral{}, p.errorf("bad date literal %q: %v", s, perr)
  3171  	}
  3172  	// TODO: Enforce valid range.
  3173  	return DateLiteral(d), nil
  3174  }
  3175  
  3176  // TODO: A manual parser is probably better than this.
  3177  // There are a lot of variations that this does not handle.
  3178  var timestampFormats = []string{
  3179  	// 'YYYY-[M]M-[D]D [[H]H:[M]M:[S]S[.DDDDDD] [timezone]]'
  3180  	"2006-01-02",
  3181  	"2006-01-02 15:04:05",
  3182  	"2006-01-02 15:04:05.000000",
  3183  	"2006-01-02 15:04:05-07:00",
  3184  	"2006-01-02 15:04:05.000000-07:00",
  3185  }
  3186  
  3187  var defaultLocation = func() *time.Location {
  3188  	// The docs say "America/Los_Angeles" is the default.
  3189  	// Use that if we can load it, but fall back on UTC if we don't have timezone data.
  3190  	loc, err := time.LoadLocation("America/Los_Angeles")
  3191  	if err == nil {
  3192  		return loc
  3193  	}
  3194  	return time.UTC
  3195  }()
  3196  
  3197  func (p *parser) parseTimestampLit() (TimestampLiteral, *parseError) {
  3198  	if err := p.expect("TIMESTAMP"); err != nil {
  3199  		return TimestampLiteral{}, err
  3200  	}
  3201  	s, err := p.parseStringLit()
  3202  	if err != nil {
  3203  		return TimestampLiteral{}, err
  3204  	}
  3205  	for _, format := range timestampFormats {
  3206  		t, err := time.ParseInLocation(format, string(s), defaultLocation)
  3207  		if err == nil {
  3208  			// TODO: Enforce valid range.
  3209  			return TimestampLiteral(t), nil
  3210  		}
  3211  	}
  3212  	return TimestampLiteral{}, p.errorf("invalid timestamp literal %q", s)
  3213  }
  3214  
  3215  func (p *parser) parseJSONLit() (JSONLiteral, *parseError) {
  3216  	if err := p.expect("JSON"); err != nil {
  3217  		return JSONLiteral{}, err
  3218  	}
  3219  	s, err := p.parseStringLit()
  3220  	if err != nil {
  3221  		return JSONLiteral{}, err
  3222  	}
  3223  	// It is not guaranteed that the returned JSONLiteral is a valid JSON document
  3224  	// to avoid error due to parsing SQL generated with an invalid JSONLiteral like JSONLiteral("")
  3225  	return JSONLiteral(s), nil
  3226  }
  3227  
  3228  func (p *parser) parseStringLit() (StringLiteral, *parseError) {
  3229  	tok := p.next()
  3230  	if tok.err != nil {
  3231  		return "", tok.err
  3232  	}
  3233  	if tok.typ != stringToken {
  3234  		return "", p.errorf("got %q, want string literal", tok.value)
  3235  	}
  3236  	return StringLiteral(tok.string), nil
  3237  }
  3238  
  3239  func (p *parser) parsePathExp() (PathExp, *parseError) {
  3240  	var pe PathExp
  3241  	for {
  3242  		tok := p.next()
  3243  		if tok.err != nil {
  3244  			return nil, tok.err
  3245  		}
  3246  		switch tok.typ {
  3247  		case quotedID:
  3248  			pe = append(pe, ID(tok.string))
  3249  		case unquotedID:
  3250  			pe = append(pe, ID(tok.value))
  3251  		default:
  3252  			// TODO: Is this correct?
  3253  			return nil, p.errorf("expected identifer")
  3254  		}
  3255  		if !p.eat(".") {
  3256  			break
  3257  		}
  3258  	}
  3259  	return pe, nil
  3260  }
  3261  
  3262  func (p *parser) parseBoolExpr() (BoolExpr, *parseError) {
  3263  	expr, err := p.parseExpr()
  3264  	if err != nil {
  3265  		return nil, err
  3266  	}
  3267  	be, ok := expr.(BoolExpr)
  3268  	if !ok {
  3269  		return nil, p.errorf("got non-bool expression %T", expr)
  3270  	}
  3271  	return be, nil
  3272  }
  3273  
  3274  func (p *parser) parseAlias() (ID, *parseError) {
  3275  	// The docs don't specify what lexical token is valid for an alias,
  3276  	// but it seems likely that it is an identifier.
  3277  	return p.parseTableOrIndexOrColumnName()
  3278  }
  3279  
  3280  func (p *parser) parseHints(hints map[string]string) (map[string]string, *parseError) {
  3281  	if hints == nil {
  3282  		hints = map[string]string{}
  3283  	}
  3284  	if err := p.expect("{"); err != nil {
  3285  		return nil, err
  3286  	}
  3287  	for {
  3288  		if p.sniff("}") {
  3289  			break
  3290  		}
  3291  		tok := p.next()
  3292  		if tok.err != nil {
  3293  			return nil, tok.err
  3294  		}
  3295  		k := tok.value
  3296  		if err := p.expect("="); err != nil {
  3297  			return nil, err
  3298  		}
  3299  		tok = p.next()
  3300  		if tok.err != nil {
  3301  			return nil, tok.err
  3302  		}
  3303  		v := tok.value
  3304  		hints[k] = v
  3305  		if !p.eat(",") {
  3306  			break
  3307  		}
  3308  	}
  3309  	if err := p.expect("}"); err != nil {
  3310  		return nil, err
  3311  	}
  3312  	return hints, nil
  3313  }
  3314  
  3315  func (p *parser) parseTableOrIndexOrColumnName() (ID, *parseError) {
  3316  	/*
  3317  		table_name and column_name and index_name:
  3318  				{a—z|A—Z}[{a—z|A—Z|0—9|_}+]
  3319  	*/
  3320  
  3321  	tok := p.next()
  3322  	if tok.err != nil {
  3323  		return "", tok.err
  3324  	}
  3325  	switch tok.typ {
  3326  	case quotedID:
  3327  		return ID(tok.string), nil
  3328  	case unquotedID:
  3329  		// TODO: enforce restrictions
  3330  		return ID(tok.value), nil
  3331  	default:
  3332  		return "", p.errorf("expected identifier")
  3333  	}
  3334  }
  3335  
  3336  func (p *parser) parseOnDelete() (OnDelete, *parseError) {
  3337  	/*
  3338  		CASCADE
  3339  		NO ACTION
  3340  	*/
  3341  
  3342  	tok := p.next()
  3343  	if tok.err != nil {
  3344  		return 0, tok.err
  3345  	}
  3346  	if tok.caseEqual("CASCADE") {
  3347  		return CascadeOnDelete, nil
  3348  	}
  3349  	if !tok.caseEqual("NO") {
  3350  		return 0, p.errorf("got %q, want NO or CASCADE", tok.value)
  3351  	}
  3352  	if err := p.expect("ACTION"); err != nil {
  3353  		return 0, err
  3354  	}
  3355  	return NoActionOnDelete, nil
  3356  }
  3357  
  3358  func (p *parser) parseRowDeletionPolicy() (RowDeletionPolicy, *parseError) {
  3359  	if err := p.expect("(", "OLDER_THAN", "("); err != nil {
  3360  		return RowDeletionPolicy{}, err
  3361  	}
  3362  	cname, err := p.parseTableOrIndexOrColumnName()
  3363  	if err != nil {
  3364  		return RowDeletionPolicy{}, err
  3365  	}
  3366  	if err := p.expect(",", "INTERVAL"); err != nil {
  3367  		return RowDeletionPolicy{}, err
  3368  	}
  3369  	tok := p.next()
  3370  	if tok.err != nil {
  3371  		return RowDeletionPolicy{}, tok.err
  3372  	}
  3373  	if tok.typ != int64Token {
  3374  		return RowDeletionPolicy{}, p.errorf("got %q, expected int64 token", tok.value)
  3375  	}
  3376  	n, serr := strconv.ParseInt(tok.value, tok.int64Base, 64)
  3377  	if serr != nil {
  3378  		return RowDeletionPolicy{}, p.errorf("%v", serr)
  3379  	}
  3380  	if err := p.expect("DAY", ")", ")"); err != nil {
  3381  		return RowDeletionPolicy{}, err
  3382  	}
  3383  	return RowDeletionPolicy{
  3384  		Column:  cname,
  3385  		NumDays: n,
  3386  	}, nil
  3387  }
  3388  
  3389  // parseCommaList parses a comma-separated list enclosed by bra and ket,
  3390  // delegating to f for the individual element parsing.
  3391  // Only invoke this with symbols as bra/ket; they are matched literally, not case insensitively.
  3392  func (p *parser) parseCommaList(bra, ket string, f func(*parser) *parseError) *parseError {
  3393  	if err := p.expect(bra); err != nil {
  3394  		return err
  3395  	}
  3396  	for {
  3397  		if p.eat(ket) {
  3398  			return nil
  3399  		}
  3400  
  3401  		err := f(p)
  3402  		if err != nil {
  3403  			return err
  3404  		}
  3405  
  3406  		// ket or "," should be next.
  3407  		tok := p.next()
  3408  		if tok.err != nil {
  3409  			return err
  3410  		}
  3411  		if tok.value == ket {
  3412  			return nil
  3413  		} else if tok.value == "," {
  3414  			continue
  3415  		} else {
  3416  			return p.errorf(`got %q, want %q or ","`, tok.value, ket)
  3417  		}
  3418  	}
  3419  }