github.com/guyezi/gofrontend@v0.0.0-20200228202240-7a62a49e62c0/libgo/go/text/template/parse/lex_test.go (about)

     1  // Copyright 2011 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  package parse
     6  
     7  import (
     8  	"fmt"
     9  	"testing"
    10  )
    11  
    12  // Make the types prettyprint.
    13  var itemName = map[itemType]string{
    14  	itemError:        "error",
    15  	itemBool:         "bool",
    16  	itemChar:         "char",
    17  	itemCharConstant: "charconst",
    18  	itemComplex:      "complex",
    19  	itemDeclare:      ":=",
    20  	itemEOF:          "EOF",
    21  	itemField:        "field",
    22  	itemIdentifier:   "identifier",
    23  	itemLeftDelim:    "left delim",
    24  	itemLeftParen:    "(",
    25  	itemNumber:       "number",
    26  	itemPipe:         "pipe",
    27  	itemRawString:    "raw string",
    28  	itemRightDelim:   "right delim",
    29  	itemRightParen:   ")",
    30  	itemSpace:        "space",
    31  	itemString:       "string",
    32  	itemVariable:     "variable",
    33  
    34  	// keywords
    35  	itemDot:      ".",
    36  	itemBlock:    "block",
    37  	itemDefine:   "define",
    38  	itemElse:     "else",
    39  	itemIf:       "if",
    40  	itemEnd:      "end",
    41  	itemNil:      "nil",
    42  	itemRange:    "range",
    43  	itemTemplate: "template",
    44  	itemWith:     "with",
    45  }
    46  
    47  func (i itemType) String() string {
    48  	s := itemName[i]
    49  	if s == "" {
    50  		return fmt.Sprintf("item%d", int(i))
    51  	}
    52  	return s
    53  }
    54  
    55  type lexTest struct {
    56  	name  string
    57  	input string
    58  	items []item
    59  }
    60  
    61  func mkItem(typ itemType, text string) item {
    62  	return item{
    63  		typ: typ,
    64  		val: text,
    65  	}
    66  }
    67  
    68  var (
    69  	tDot        = mkItem(itemDot, ".")
    70  	tBlock      = mkItem(itemBlock, "block")
    71  	tEOF        = mkItem(itemEOF, "")
    72  	tFor        = mkItem(itemIdentifier, "for")
    73  	tLeft       = mkItem(itemLeftDelim, "{{")
    74  	tLpar       = mkItem(itemLeftParen, "(")
    75  	tPipe       = mkItem(itemPipe, "|")
    76  	tQuote      = mkItem(itemString, `"abc \n\t\" "`)
    77  	tRange      = mkItem(itemRange, "range")
    78  	tRight      = mkItem(itemRightDelim, "}}")
    79  	tRpar       = mkItem(itemRightParen, ")")
    80  	tSpace      = mkItem(itemSpace, " ")
    81  	raw         = "`" + `abc\n\t\" ` + "`"
    82  	rawNL       = "`now is{{\n}}the time`" // Contains newline inside raw quote.
    83  	tRawQuote   = mkItem(itemRawString, raw)
    84  	tRawQuoteNL = mkItem(itemRawString, rawNL)
    85  )
    86  
    87  var lexTests = []lexTest{
    88  	{"empty", "", []item{tEOF}},
    89  	{"spaces", " \t\n", []item{mkItem(itemText, " \t\n"), tEOF}},
    90  	{"text", `now is the time`, []item{mkItem(itemText, "now is the time"), tEOF}},
    91  	{"text with comment", "hello-{{/* this is a comment */}}-world", []item{
    92  		mkItem(itemText, "hello-"),
    93  		mkItem(itemText, "-world"),
    94  		tEOF,
    95  	}},
    96  	{"punctuation", "{{,@% }}", []item{
    97  		tLeft,
    98  		mkItem(itemChar, ","),
    99  		mkItem(itemChar, "@"),
   100  		mkItem(itemChar, "%"),
   101  		tSpace,
   102  		tRight,
   103  		tEOF,
   104  	}},
   105  	{"parens", "{{((3))}}", []item{
   106  		tLeft,
   107  		tLpar,
   108  		tLpar,
   109  		mkItem(itemNumber, "3"),
   110  		tRpar,
   111  		tRpar,
   112  		tRight,
   113  		tEOF,
   114  	}},
   115  	{"empty action", `{{}}`, []item{tLeft, tRight, tEOF}},
   116  	{"for", `{{for}}`, []item{tLeft, tFor, tRight, tEOF}},
   117  	{"block", `{{block "foo" .}}`, []item{
   118  		tLeft, tBlock, tSpace, mkItem(itemString, `"foo"`), tSpace, tDot, tRight, tEOF,
   119  	}},
   120  	{"quote", `{{"abc \n\t\" "}}`, []item{tLeft, tQuote, tRight, tEOF}},
   121  	{"raw quote", "{{" + raw + "}}", []item{tLeft, tRawQuote, tRight, tEOF}},
   122  	{"raw quote with newline", "{{" + rawNL + "}}", []item{tLeft, tRawQuoteNL, tRight, tEOF}},
   123  	{"numbers", "{{1 02 0x14 0X14 -7.2i 1e3 1E3 +1.2e-4 4.2i 1+2i 1_2 0x1.e_fp4 0X1.E_FP4}}", []item{
   124  		tLeft,
   125  		mkItem(itemNumber, "1"),
   126  		tSpace,
   127  		mkItem(itemNumber, "02"),
   128  		tSpace,
   129  		mkItem(itemNumber, "0x14"),
   130  		tSpace,
   131  		mkItem(itemNumber, "0X14"),
   132  		tSpace,
   133  		mkItem(itemNumber, "-7.2i"),
   134  		tSpace,
   135  		mkItem(itemNumber, "1e3"),
   136  		tSpace,
   137  		mkItem(itemNumber, "1E3"),
   138  		tSpace,
   139  		mkItem(itemNumber, "+1.2e-4"),
   140  		tSpace,
   141  		mkItem(itemNumber, "4.2i"),
   142  		tSpace,
   143  		mkItem(itemComplex, "1+2i"),
   144  		tSpace,
   145  		mkItem(itemNumber, "1_2"),
   146  		tSpace,
   147  		mkItem(itemNumber, "0x1.e_fp4"),
   148  		tSpace,
   149  		mkItem(itemNumber, "0X1.E_FP4"),
   150  		tRight,
   151  		tEOF,
   152  	}},
   153  	{"characters", `{{'a' '\n' '\'' '\\' '\u00FF' '\xFF' '本'}}`, []item{
   154  		tLeft,
   155  		mkItem(itemCharConstant, `'a'`),
   156  		tSpace,
   157  		mkItem(itemCharConstant, `'\n'`),
   158  		tSpace,
   159  		mkItem(itemCharConstant, `'\''`),
   160  		tSpace,
   161  		mkItem(itemCharConstant, `'\\'`),
   162  		tSpace,
   163  		mkItem(itemCharConstant, `'\u00FF'`),
   164  		tSpace,
   165  		mkItem(itemCharConstant, `'\xFF'`),
   166  		tSpace,
   167  		mkItem(itemCharConstant, `'本'`),
   168  		tRight,
   169  		tEOF,
   170  	}},
   171  	{"bools", "{{true false}}", []item{
   172  		tLeft,
   173  		mkItem(itemBool, "true"),
   174  		tSpace,
   175  		mkItem(itemBool, "false"),
   176  		tRight,
   177  		tEOF,
   178  	}},
   179  	{"dot", "{{.}}", []item{
   180  		tLeft,
   181  		tDot,
   182  		tRight,
   183  		tEOF,
   184  	}},
   185  	{"nil", "{{nil}}", []item{
   186  		tLeft,
   187  		mkItem(itemNil, "nil"),
   188  		tRight,
   189  		tEOF,
   190  	}},
   191  	{"dots", "{{.x . .2 .x.y.z}}", []item{
   192  		tLeft,
   193  		mkItem(itemField, ".x"),
   194  		tSpace,
   195  		tDot,
   196  		tSpace,
   197  		mkItem(itemNumber, ".2"),
   198  		tSpace,
   199  		mkItem(itemField, ".x"),
   200  		mkItem(itemField, ".y"),
   201  		mkItem(itemField, ".z"),
   202  		tRight,
   203  		tEOF,
   204  	}},
   205  	{"keywords", "{{range if else end with}}", []item{
   206  		tLeft,
   207  		mkItem(itemRange, "range"),
   208  		tSpace,
   209  		mkItem(itemIf, "if"),
   210  		tSpace,
   211  		mkItem(itemElse, "else"),
   212  		tSpace,
   213  		mkItem(itemEnd, "end"),
   214  		tSpace,
   215  		mkItem(itemWith, "with"),
   216  		tRight,
   217  		tEOF,
   218  	}},
   219  	{"variables", "{{$c := printf $ $hello $23 $ $var.Field .Method}}", []item{
   220  		tLeft,
   221  		mkItem(itemVariable, "$c"),
   222  		tSpace,
   223  		mkItem(itemDeclare, ":="),
   224  		tSpace,
   225  		mkItem(itemIdentifier, "printf"),
   226  		tSpace,
   227  		mkItem(itemVariable, "$"),
   228  		tSpace,
   229  		mkItem(itemVariable, "$hello"),
   230  		tSpace,
   231  		mkItem(itemVariable, "$23"),
   232  		tSpace,
   233  		mkItem(itemVariable, "$"),
   234  		tSpace,
   235  		mkItem(itemVariable, "$var"),
   236  		mkItem(itemField, ".Field"),
   237  		tSpace,
   238  		mkItem(itemField, ".Method"),
   239  		tRight,
   240  		tEOF,
   241  	}},
   242  	{"variable invocation", "{{$x 23}}", []item{
   243  		tLeft,
   244  		mkItem(itemVariable, "$x"),
   245  		tSpace,
   246  		mkItem(itemNumber, "23"),
   247  		tRight,
   248  		tEOF,
   249  	}},
   250  	{"pipeline", `intro {{echo hi 1.2 |noargs|args 1 "hi"}} outro`, []item{
   251  		mkItem(itemText, "intro "),
   252  		tLeft,
   253  		mkItem(itemIdentifier, "echo"),
   254  		tSpace,
   255  		mkItem(itemIdentifier, "hi"),
   256  		tSpace,
   257  		mkItem(itemNumber, "1.2"),
   258  		tSpace,
   259  		tPipe,
   260  		mkItem(itemIdentifier, "noargs"),
   261  		tPipe,
   262  		mkItem(itemIdentifier, "args"),
   263  		tSpace,
   264  		mkItem(itemNumber, "1"),
   265  		tSpace,
   266  		mkItem(itemString, `"hi"`),
   267  		tRight,
   268  		mkItem(itemText, " outro"),
   269  		tEOF,
   270  	}},
   271  	{"declaration", "{{$v := 3}}", []item{
   272  		tLeft,
   273  		mkItem(itemVariable, "$v"),
   274  		tSpace,
   275  		mkItem(itemDeclare, ":="),
   276  		tSpace,
   277  		mkItem(itemNumber, "3"),
   278  		tRight,
   279  		tEOF,
   280  	}},
   281  	{"2 declarations", "{{$v , $w := 3}}", []item{
   282  		tLeft,
   283  		mkItem(itemVariable, "$v"),
   284  		tSpace,
   285  		mkItem(itemChar, ","),
   286  		tSpace,
   287  		mkItem(itemVariable, "$w"),
   288  		tSpace,
   289  		mkItem(itemDeclare, ":="),
   290  		tSpace,
   291  		mkItem(itemNumber, "3"),
   292  		tRight,
   293  		tEOF,
   294  	}},
   295  	{"field of parenthesized expression", "{{(.X).Y}}", []item{
   296  		tLeft,
   297  		tLpar,
   298  		mkItem(itemField, ".X"),
   299  		tRpar,
   300  		mkItem(itemField, ".Y"),
   301  		tRight,
   302  		tEOF,
   303  	}},
   304  	{"trimming spaces before and after", "hello- {{- 3 -}} -world", []item{
   305  		mkItem(itemText, "hello-"),
   306  		tLeft,
   307  		mkItem(itemNumber, "3"),
   308  		tRight,
   309  		mkItem(itemText, "-world"),
   310  		tEOF,
   311  	}},
   312  	{"trimming spaces before and after comment", "hello- {{- /* hello */ -}} -world", []item{
   313  		mkItem(itemText, "hello-"),
   314  		mkItem(itemText, "-world"),
   315  		tEOF,
   316  	}},
   317  	// errors
   318  	{"badchar", "#{{\x01}}", []item{
   319  		mkItem(itemText, "#"),
   320  		tLeft,
   321  		mkItem(itemError, "unrecognized character in action: U+0001"),
   322  	}},
   323  	{"unclosed action", "{{\n}}", []item{
   324  		tLeft,
   325  		mkItem(itemError, "unclosed action"),
   326  	}},
   327  	{"EOF in action", "{{range", []item{
   328  		tLeft,
   329  		tRange,
   330  		mkItem(itemError, "unclosed action"),
   331  	}},
   332  	{"unclosed quote", "{{\"\n\"}}", []item{
   333  		tLeft,
   334  		mkItem(itemError, "unterminated quoted string"),
   335  	}},
   336  	{"unclosed raw quote", "{{`xx}}", []item{
   337  		tLeft,
   338  		mkItem(itemError, "unterminated raw quoted string"),
   339  	}},
   340  	{"unclosed char constant", "{{'\n}}", []item{
   341  		tLeft,
   342  		mkItem(itemError, "unterminated character constant"),
   343  	}},
   344  	{"bad number", "{{3k}}", []item{
   345  		tLeft,
   346  		mkItem(itemError, `bad number syntax: "3k"`),
   347  	}},
   348  	{"unclosed paren", "{{(3}}", []item{
   349  		tLeft,
   350  		tLpar,
   351  		mkItem(itemNumber, "3"),
   352  		mkItem(itemError, `unclosed left paren`),
   353  	}},
   354  	{"extra right paren", "{{3)}}", []item{
   355  		tLeft,
   356  		mkItem(itemNumber, "3"),
   357  		tRpar,
   358  		mkItem(itemError, `unexpected right paren U+0029 ')'`),
   359  	}},
   360  
   361  	// Fixed bugs
   362  	// Many elements in an action blew the lookahead until
   363  	// we made lexInsideAction not loop.
   364  	{"long pipeline deadlock", "{{|||||}}", []item{
   365  		tLeft,
   366  		tPipe,
   367  		tPipe,
   368  		tPipe,
   369  		tPipe,
   370  		tPipe,
   371  		tRight,
   372  		tEOF,
   373  	}},
   374  	{"text with bad comment", "hello-{{/*/}}-world", []item{
   375  		mkItem(itemText, "hello-"),
   376  		mkItem(itemError, `unclosed comment`),
   377  	}},
   378  	{"text with comment close separated from delim", "hello-{{/* */ }}-world", []item{
   379  		mkItem(itemText, "hello-"),
   380  		mkItem(itemError, `comment ends before closing delimiter`),
   381  	}},
   382  	// This one is an error that we can't catch because it breaks templates with
   383  	// minimized JavaScript. Should have fixed it before Go 1.1.
   384  	{"unmatched right delimiter", "hello-{.}}-world", []item{
   385  		mkItem(itemText, "hello-{.}}-world"),
   386  		tEOF,
   387  	}},
   388  }
   389  
   390  // collect gathers the emitted items into a slice.
   391  func collect(t *lexTest, left, right string) (items []item) {
   392  	l := lex(t.name, t.input, left, right)
   393  	for {
   394  		item := l.nextItem()
   395  		items = append(items, item)
   396  		if item.typ == itemEOF || item.typ == itemError {
   397  			break
   398  		}
   399  	}
   400  	return
   401  }
   402  
   403  func equal(i1, i2 []item, checkPos bool) bool {
   404  	if len(i1) != len(i2) {
   405  		return false
   406  	}
   407  	for k := range i1 {
   408  		if i1[k].typ != i2[k].typ {
   409  			return false
   410  		}
   411  		if i1[k].val != i2[k].val {
   412  			return false
   413  		}
   414  		if checkPos && i1[k].pos != i2[k].pos {
   415  			return false
   416  		}
   417  		if checkPos && i1[k].line != i2[k].line {
   418  			return false
   419  		}
   420  	}
   421  	return true
   422  }
   423  
   424  func TestLex(t *testing.T) {
   425  	for _, test := range lexTests {
   426  		items := collect(&test, "", "")
   427  		if !equal(items, test.items, false) {
   428  			t.Errorf("%s: got\n\t%+v\nexpected\n\t%v", test.name, items, test.items)
   429  		}
   430  	}
   431  }
   432  
   433  // Some easy cases from above, but with delimiters $$ and @@
   434  var lexDelimTests = []lexTest{
   435  	{"punctuation", "$$,@%{{}}@@", []item{
   436  		tLeftDelim,
   437  		mkItem(itemChar, ","),
   438  		mkItem(itemChar, "@"),
   439  		mkItem(itemChar, "%"),
   440  		mkItem(itemChar, "{"),
   441  		mkItem(itemChar, "{"),
   442  		mkItem(itemChar, "}"),
   443  		mkItem(itemChar, "}"),
   444  		tRightDelim,
   445  		tEOF,
   446  	}},
   447  	{"empty action", `$$@@`, []item{tLeftDelim, tRightDelim, tEOF}},
   448  	{"for", `$$for@@`, []item{tLeftDelim, tFor, tRightDelim, tEOF}},
   449  	{"quote", `$$"abc \n\t\" "@@`, []item{tLeftDelim, tQuote, tRightDelim, tEOF}},
   450  	{"raw quote", "$$" + raw + "@@", []item{tLeftDelim, tRawQuote, tRightDelim, tEOF}},
   451  }
   452  
   453  var (
   454  	tLeftDelim  = mkItem(itemLeftDelim, "$$")
   455  	tRightDelim = mkItem(itemRightDelim, "@@")
   456  )
   457  
   458  func TestDelims(t *testing.T) {
   459  	for _, test := range lexDelimTests {
   460  		items := collect(&test, "$$", "@@")
   461  		if !equal(items, test.items, false) {
   462  			t.Errorf("%s: got\n\t%v\nexpected\n\t%v", test.name, items, test.items)
   463  		}
   464  	}
   465  }
   466  
   467  var lexPosTests = []lexTest{
   468  	{"empty", "", []item{{itemEOF, 0, "", 1}}},
   469  	{"punctuation", "{{,@%#}}", []item{
   470  		{itemLeftDelim, 0, "{{", 1},
   471  		{itemChar, 2, ",", 1},
   472  		{itemChar, 3, "@", 1},
   473  		{itemChar, 4, "%", 1},
   474  		{itemChar, 5, "#", 1},
   475  		{itemRightDelim, 6, "}}", 1},
   476  		{itemEOF, 8, "", 1},
   477  	}},
   478  	{"sample", "0123{{hello}}xyz", []item{
   479  		{itemText, 0, "0123", 1},
   480  		{itemLeftDelim, 4, "{{", 1},
   481  		{itemIdentifier, 6, "hello", 1},
   482  		{itemRightDelim, 11, "}}", 1},
   483  		{itemText, 13, "xyz", 1},
   484  		{itemEOF, 16, "", 1},
   485  	}},
   486  	{"trimafter", "{{x -}}\n{{y}}", []item{
   487  		{itemLeftDelim, 0, "{{", 1},
   488  		{itemIdentifier, 2, "x", 1},
   489  		{itemRightDelim, 5, "}}", 1},
   490  		{itemLeftDelim, 8, "{{", 2},
   491  		{itemIdentifier, 10, "y", 2},
   492  		{itemRightDelim, 11, "}}", 2},
   493  		{itemEOF, 13, "", 2},
   494  	}},
   495  	{"trimbefore", "{{x}}\n{{- y}}", []item{
   496  		{itemLeftDelim, 0, "{{", 1},
   497  		{itemIdentifier, 2, "x", 1},
   498  		{itemRightDelim, 3, "}}", 1},
   499  		{itemLeftDelim, 6, "{{", 2},
   500  		{itemIdentifier, 10, "y", 2},
   501  		{itemRightDelim, 11, "}}", 2},
   502  		{itemEOF, 13, "", 2},
   503  	}},
   504  }
   505  
   506  // The other tests don't check position, to make the test cases easier to construct.
   507  // This one does.
   508  func TestPos(t *testing.T) {
   509  	for _, test := range lexPosTests {
   510  		items := collect(&test, "", "")
   511  		if !equal(items, test.items, true) {
   512  			t.Errorf("%s: got\n\t%v\nexpected\n\t%v", test.name, items, test.items)
   513  			if len(items) == len(test.items) {
   514  				// Detailed print; avoid item.String() to expose the position value.
   515  				for i := range items {
   516  					if !equal(items[i:i+1], test.items[i:i+1], true) {
   517  						i1 := items[i]
   518  						i2 := test.items[i]
   519  						t.Errorf("\t#%d: got {%v %d %q %d} expected {%v %d %q %d}",
   520  							i, i1.typ, i1.pos, i1.val, i1.line, i2.typ, i2.pos, i2.val, i2.line)
   521  					}
   522  				}
   523  			}
   524  		}
   525  	}
   526  }
   527  
   528  // Test that an error shuts down the lexing goroutine.
   529  func TestShutdown(t *testing.T) {
   530  	// We need to duplicate template.Parse here to hold on to the lexer.
   531  	const text = "erroneous{{define}}{{else}}1234"
   532  	lexer := lex("foo", text, "{{", "}}")
   533  	_, err := New("root").parseLexer(lexer)
   534  	if err == nil {
   535  		t.Fatalf("expected error")
   536  	}
   537  	// The error should have drained the input. Therefore, the lexer should be shut down.
   538  	token, ok := <-lexer.items
   539  	if ok {
   540  		t.Fatalf("input was not drained; got %v", token)
   541  	}
   542  }
   543  
   544  // parseLexer is a local version of parse that lets us pass in the lexer instead of building it.
   545  // We expect an error, so the tree set and funcs list are explicitly nil.
   546  func (t *Tree) parseLexer(lex *lexer) (tree *Tree, err error) {
   547  	defer t.recover(&err)
   548  	t.ParseName = t.Name
   549  	t.startParse(nil, lex, map[string]*Tree{})
   550  	t.parse()
   551  	t.add()
   552  	t.stopParse()
   553  	return t, nil
   554  }