github.com/hashicorp/hcl/v2@v2.20.0/hclsyntax/scan_tokens_test.go (about)

     1  // Copyright (c) HashiCorp, Inc.
     2  // SPDX-License-Identifier: MPL-2.0
     3  
     4  package hclsyntax
     5  
     6  import (
     7  	"testing"
     8  
     9  	"github.com/google/go-cmp/cmp"
    10  	"github.com/hashicorp/hcl/v2"
    11  )
    12  
    13  func TestScanTokens_normal(t *testing.T) {
    14  	tests := []struct {
    15  		input string
    16  		want  []Token
    17  	}{
    18  		// Empty input
    19  		{
    20  			``,
    21  			[]Token{
    22  				{
    23  					Type:  TokenEOF,
    24  					Bytes: []byte{},
    25  					Range: hcl.Range{
    26  						Start: hcl.Pos{Byte: 0, Line: 1, Column: 1},
    27  						End:   hcl.Pos{Byte: 0, Line: 1, Column: 1},
    28  					},
    29  				},
    30  			},
    31  		},
    32  		{
    33  			` `,
    34  			[]Token{
    35  				{
    36  					Type:  TokenEOF,
    37  					Bytes: []byte{},
    38  					Range: hcl.Range{
    39  						Start: hcl.Pos{Byte: 1, Line: 1, Column: 2},
    40  						End:   hcl.Pos{Byte: 1, Line: 1, Column: 2},
    41  					},
    42  				},
    43  			},
    44  		},
    45  		{
    46  			"\n\n",
    47  			[]Token{
    48  				{
    49  					Type:  TokenNewline,
    50  					Bytes: []byte("\n"),
    51  					Range: hcl.Range{
    52  						Start: hcl.Pos{Byte: 0, Line: 1, Column: 1},
    53  						End:   hcl.Pos{Byte: 1, Line: 2, Column: 1},
    54  					},
    55  				},
    56  				{
    57  					Type:  TokenNewline,
    58  					Bytes: []byte("\n"),
    59  					Range: hcl.Range{
    60  						Start: hcl.Pos{Byte: 1, Line: 2, Column: 1},
    61  						End:   hcl.Pos{Byte: 2, Line: 3, Column: 1},
    62  					},
    63  				},
    64  				{
    65  					Type:  TokenEOF,
    66  					Bytes: []byte{},
    67  					Range: hcl.Range{
    68  						Start: hcl.Pos{Byte: 2, Line: 3, Column: 1},
    69  						End:   hcl.Pos{Byte: 2, Line: 3, Column: 1},
    70  					},
    71  				},
    72  			},
    73  		},
    74  
    75  		// Byte-order mark
    76  		{
    77  			"\xef\xbb\xbf", // Leading UTF-8 byte-order mark is ignored...
    78  			[]Token{
    79  				{
    80  					Type:  TokenEOF,
    81  					Bytes: []byte{},
    82  					Range: hcl.Range{ // ...but its bytes still count when producing ranges
    83  						Start: hcl.Pos{Byte: 3, Line: 1, Column: 1},
    84  						End:   hcl.Pos{Byte: 3, Line: 1, Column: 1},
    85  					},
    86  				},
    87  			},
    88  		},
    89  		{
    90  			" \xef\xbb\xbf", // Non-leading BOM is invalid
    91  			[]Token{
    92  				{
    93  					Type:  TokenInvalid,
    94  					Bytes: utf8BOM,
    95  					Range: hcl.Range{
    96  						Start: hcl.Pos{Byte: 1, Line: 1, Column: 2},
    97  						End:   hcl.Pos{Byte: 4, Line: 1, Column: 3},
    98  					},
    99  				},
   100  				{
   101  					Type:  TokenEOF,
   102  					Bytes: []byte{},
   103  					Range: hcl.Range{
   104  						Start: hcl.Pos{Byte: 4, Line: 1, Column: 3},
   105  						End:   hcl.Pos{Byte: 4, Line: 1, Column: 3},
   106  					},
   107  				},
   108  			},
   109  		},
   110  		{
   111  			"\xfe\xff", // UTF-16 BOM is invalid
   112  			[]Token{
   113  				{
   114  					Type:  TokenBadUTF8,
   115  					Bytes: []byte{0xfe},
   116  					Range: hcl.Range{
   117  						Start: hcl.Pos{Byte: 0, Line: 1, Column: 1},
   118  						End:   hcl.Pos{Byte: 1, Line: 1, Column: 2},
   119  					},
   120  				},
   121  				{
   122  					Type:  TokenBadUTF8,
   123  					Bytes: []byte{0xff},
   124  					Range: hcl.Range{
   125  						Start: hcl.Pos{Byte: 1, Line: 1, Column: 2},
   126  						End:   hcl.Pos{Byte: 2, Line: 1, Column: 3},
   127  					},
   128  				},
   129  				{
   130  					Type:  TokenEOF,
   131  					Bytes: []byte{},
   132  					Range: hcl.Range{
   133  						Start: hcl.Pos{Byte: 2, Line: 1, Column: 3},
   134  						End:   hcl.Pos{Byte: 2, Line: 1, Column: 3},
   135  					},
   136  				},
   137  			},
   138  		},
   139  
   140  		// TokenNumberLit
   141  		{
   142  			`1`,
   143  			[]Token{
   144  				{
   145  					Type:  TokenNumberLit,
   146  					Bytes: []byte(`1`),
   147  					Range: hcl.Range{
   148  						Start: hcl.Pos{Byte: 0, Line: 1, Column: 1},
   149  						End:   hcl.Pos{Byte: 1, Line: 1, Column: 2},
   150  					},
   151  				},
   152  				{
   153  					Type:  TokenEOF,
   154  					Bytes: []byte{},
   155  					Range: hcl.Range{
   156  						Start: hcl.Pos{Byte: 1, Line: 1, Column: 2},
   157  						End:   hcl.Pos{Byte: 1, Line: 1, Column: 2},
   158  					},
   159  				},
   160  			},
   161  		},
   162  		{
   163  			`12`,
   164  			[]Token{
   165  				{
   166  					Type:  TokenNumberLit,
   167  					Bytes: []byte(`12`),
   168  					Range: hcl.Range{
   169  						Start: hcl.Pos{Byte: 0, Line: 1, Column: 1},
   170  						End:   hcl.Pos{Byte: 2, Line: 1, Column: 3},
   171  					},
   172  				},
   173  				{
   174  					Type:  TokenEOF,
   175  					Bytes: []byte{},
   176  					Range: hcl.Range{
   177  						Start: hcl.Pos{Byte: 2, Line: 1, Column: 3},
   178  						End:   hcl.Pos{Byte: 2, Line: 1, Column: 3},
   179  					},
   180  				},
   181  			},
   182  		},
   183  		{
   184  			`12.3`,
   185  			[]Token{
   186  				{
   187  					Type:  TokenNumberLit,
   188  					Bytes: []byte(`12.3`),
   189  					Range: hcl.Range{
   190  						Start: hcl.Pos{Byte: 0, Line: 1, Column: 1},
   191  						End:   hcl.Pos{Byte: 4, Line: 1, Column: 5},
   192  					},
   193  				},
   194  				{
   195  					Type:  TokenEOF,
   196  					Bytes: []byte{},
   197  					Range: hcl.Range{
   198  						Start: hcl.Pos{Byte: 4, Line: 1, Column: 5},
   199  						End:   hcl.Pos{Byte: 4, Line: 1, Column: 5},
   200  					},
   201  				},
   202  			},
   203  		},
   204  		{
   205  			`1e2`,
   206  			[]Token{
   207  				{
   208  					Type:  TokenNumberLit,
   209  					Bytes: []byte(`1e2`),
   210  					Range: hcl.Range{
   211  						Start: hcl.Pos{Byte: 0, Line: 1, Column: 1},
   212  						End:   hcl.Pos{Byte: 3, Line: 1, Column: 4},
   213  					},
   214  				},
   215  				{
   216  					Type:  TokenEOF,
   217  					Bytes: []byte{},
   218  					Range: hcl.Range{
   219  						Start: hcl.Pos{Byte: 3, Line: 1, Column: 4},
   220  						End:   hcl.Pos{Byte: 3, Line: 1, Column: 4},
   221  					},
   222  				},
   223  			},
   224  		},
   225  		{
   226  			`1e+2`,
   227  			[]Token{
   228  				{
   229  					Type:  TokenNumberLit,
   230  					Bytes: []byte(`1e+2`),
   231  					Range: hcl.Range{
   232  						Start: hcl.Pos{Byte: 0, Line: 1, Column: 1},
   233  						End:   hcl.Pos{Byte: 4, Line: 1, Column: 5},
   234  					},
   235  				},
   236  				{
   237  					Type:  TokenEOF,
   238  					Bytes: []byte{},
   239  					Range: hcl.Range{
   240  						Start: hcl.Pos{Byte: 4, Line: 1, Column: 5},
   241  						End:   hcl.Pos{Byte: 4, Line: 1, Column: 5},
   242  					},
   243  				},
   244  			},
   245  		},
   246  
   247  		// TokenIdent
   248  		{
   249  			`hello`,
   250  			[]Token{
   251  				{
   252  					Type:  TokenIdent,
   253  					Bytes: []byte(`hello`),
   254  					Range: hcl.Range{
   255  						Start: hcl.Pos{Byte: 0, Line: 1, Column: 1},
   256  						End:   hcl.Pos{Byte: 5, Line: 1, Column: 6},
   257  					},
   258  				},
   259  				{
   260  					Type:  TokenEOF,
   261  					Bytes: []byte{},
   262  					Range: hcl.Range{
   263  						Start: hcl.Pos{Byte: 5, Line: 1, Column: 6},
   264  						End:   hcl.Pos{Byte: 5, Line: 1, Column: 6},
   265  					},
   266  				},
   267  			},
   268  		},
   269  		{
   270  			`_ello`,
   271  			[]Token{
   272  				{
   273  					Type:  TokenIdent,
   274  					Bytes: []byte(`_ello`),
   275  					Range: hcl.Range{
   276  						Start: hcl.Pos{Byte: 0, Line: 1, Column: 1},
   277  						End:   hcl.Pos{Byte: 5, Line: 1, Column: 6},
   278  					},
   279  				},
   280  				{
   281  					Type:  TokenEOF,
   282  					Bytes: []byte{},
   283  					Range: hcl.Range{
   284  						Start: hcl.Pos{Byte: 5, Line: 1, Column: 6},
   285  						End:   hcl.Pos{Byte: 5, Line: 1, Column: 6},
   286  					},
   287  				},
   288  			},
   289  		},
   290  		{
   291  			`hel_o`,
   292  			[]Token{
   293  				{
   294  					Type:  TokenIdent,
   295  					Bytes: []byte(`hel_o`),
   296  					Range: hcl.Range{
   297  						Start: hcl.Pos{Byte: 0, Line: 1, Column: 1},
   298  						End:   hcl.Pos{Byte: 5, Line: 1, Column: 6},
   299  					},
   300  				},
   301  				{
   302  					Type:  TokenEOF,
   303  					Bytes: []byte{},
   304  					Range: hcl.Range{
   305  						Start: hcl.Pos{Byte: 5, Line: 1, Column: 6},
   306  						End:   hcl.Pos{Byte: 5, Line: 1, Column: 6},
   307  					},
   308  				},
   309  			},
   310  		},
   311  		{
   312  			`hel-o`,
   313  			[]Token{
   314  				{
   315  					Type:  TokenIdent,
   316  					Bytes: []byte(`hel-o`),
   317  					Range: hcl.Range{
   318  						Start: hcl.Pos{Byte: 0, Line: 1, Column: 1},
   319  						End:   hcl.Pos{Byte: 5, Line: 1, Column: 6},
   320  					},
   321  				},
   322  				{
   323  					Type:  TokenEOF,
   324  					Bytes: []byte{},
   325  					Range: hcl.Range{
   326  						Start: hcl.Pos{Byte: 5, Line: 1, Column: 6},
   327  						End:   hcl.Pos{Byte: 5, Line: 1, Column: 6},
   328  					},
   329  				},
   330  			},
   331  		},
   332  		{
   333  			`h3ll0`,
   334  			[]Token{
   335  				{
   336  					Type:  TokenIdent,
   337  					Bytes: []byte(`h3ll0`),
   338  					Range: hcl.Range{
   339  						Start: hcl.Pos{Byte: 0, Line: 1, Column: 1},
   340  						End:   hcl.Pos{Byte: 5, Line: 1, Column: 6},
   341  					},
   342  				},
   343  				{
   344  					Type:  TokenEOF,
   345  					Bytes: []byte{},
   346  					Range: hcl.Range{
   347  						Start: hcl.Pos{Byte: 5, Line: 1, Column: 6},
   348  						End:   hcl.Pos{Byte: 5, Line: 1, Column: 6},
   349  					},
   350  				},
   351  			},
   352  		},
   353  		{
   354  			`héllo`, // combining acute accent
   355  			[]Token{
   356  				{
   357  					Type:  TokenIdent,
   358  					Bytes: []byte(`héllo`),
   359  					Range: hcl.Range{
   360  						Start: hcl.Pos{Byte: 0, Line: 1, Column: 1},
   361  						End:   hcl.Pos{Byte: 7, Line: 1, Column: 6},
   362  					},
   363  				},
   364  				{
   365  					Type:  TokenEOF,
   366  					Bytes: []byte{},
   367  					Range: hcl.Range{
   368  						Start: hcl.Pos{Byte: 7, Line: 1, Column: 6},
   369  						End:   hcl.Pos{Byte: 7, Line: 1, Column: 6},
   370  					},
   371  				},
   372  			},
   373  		},
   374  
   375  		// TokenDoubleColon and associated TokenIdent
   376  		{
   377  			`::`,
   378  			[]Token{
   379  				{
   380  					Type:  TokenDoubleColon,
   381  					Bytes: []byte(`::`),
   382  					Range: hcl.Range{
   383  						Start: hcl.Pos{Byte: 0, Line: 1, Column: 1},
   384  						End:   hcl.Pos{Byte: 2, Line: 1, Column: 3},
   385  					},
   386  				},
   387  				{
   388  					Type:  TokenEOF,
   389  					Bytes: []byte{},
   390  					Range: hcl.Range{
   391  						Start: hcl.Pos{Byte: 2, Line: 1, Column: 3},
   392  						End:   hcl.Pos{Byte: 2, Line: 1, Column: 3},
   393  					},
   394  				},
   395  			},
   396  		},
   397  		{
   398  			`a::b`,
   399  			[]Token{
   400  				{
   401  					Type:  TokenIdent,
   402  					Bytes: []byte(`a`),
   403  					Range: hcl.Range{
   404  						Start: hcl.Pos{Byte: 0, Line: 1, Column: 1},
   405  						End:   hcl.Pos{Byte: 1, Line: 1, Column: 2},
   406  					},
   407  				},
   408  				{
   409  					Type:  TokenDoubleColon,
   410  					Bytes: []byte(`::`),
   411  					Range: hcl.Range{
   412  						Start: hcl.Pos{Byte: 1, Line: 1, Column: 2},
   413  						End:   hcl.Pos{Byte: 3, Line: 1, Column: 4},
   414  					},
   415  				},
   416  				{
   417  					Type:  TokenIdent,
   418  					Bytes: []byte(`b`),
   419  					Range: hcl.Range{
   420  						Start: hcl.Pos{Byte: 3, Line: 1, Column: 4},
   421  						End:   hcl.Pos{Byte: 4, Line: 1, Column: 5},
   422  					},
   423  				},
   424  				{
   425  					Type:  TokenEOF,
   426  					Bytes: []byte{},
   427  					Range: hcl.Range{
   428  						Start: hcl.Pos{Byte: 4, Line: 1, Column: 5},
   429  						End:   hcl.Pos{Byte: 4, Line: 1, Column: 5},
   430  					},
   431  				},
   432  			},
   433  		},
   434  
   435  		// Literal-only Templates (string literals, effectively)
   436  		{
   437  			`""`,
   438  			[]Token{
   439  				{
   440  					Type:  TokenOQuote,
   441  					Bytes: []byte(`"`),
   442  					Range: hcl.Range{
   443  						Start: hcl.Pos{Byte: 0, Line: 1, Column: 1},
   444  						End:   hcl.Pos{Byte: 1, Line: 1, Column: 2},
   445  					},
   446  				},
   447  				{
   448  					Type:  TokenCQuote,
   449  					Bytes: []byte(`"`),
   450  					Range: hcl.Range{
   451  						Start: hcl.Pos{Byte: 1, Line: 1, Column: 2},
   452  						End:   hcl.Pos{Byte: 2, Line: 1, Column: 3},
   453  					},
   454  				},
   455  				{
   456  					Type:  TokenEOF,
   457  					Bytes: []byte{},
   458  					Range: hcl.Range{
   459  						Start: hcl.Pos{Byte: 2, Line: 1, Column: 3},
   460  						End:   hcl.Pos{Byte: 2, Line: 1, Column: 3},
   461  					},
   462  				},
   463  			},
   464  		},
   465  		{
   466  			`"hello"`,
   467  			[]Token{
   468  				{
   469  					Type:  TokenOQuote,
   470  					Bytes: []byte(`"`),
   471  					Range: hcl.Range{
   472  						Start: hcl.Pos{Byte: 0, Line: 1, Column: 1},
   473  						End:   hcl.Pos{Byte: 1, Line: 1, Column: 2},
   474  					},
   475  				},
   476  				{
   477  					Type:  TokenQuotedLit,
   478  					Bytes: []byte(`hello`),
   479  					Range: hcl.Range{
   480  						Start: hcl.Pos{Byte: 1, Line: 1, Column: 2},
   481  						End:   hcl.Pos{Byte: 6, Line: 1, Column: 7},
   482  					},
   483  				},
   484  				{
   485  					Type:  TokenCQuote,
   486  					Bytes: []byte(`"`),
   487  					Range: hcl.Range{
   488  						Start: hcl.Pos{Byte: 6, Line: 1, Column: 7},
   489  						End:   hcl.Pos{Byte: 7, Line: 1, Column: 8},
   490  					},
   491  				},
   492  				{
   493  					Type:  TokenEOF,
   494  					Bytes: []byte{},
   495  					Range: hcl.Range{
   496  						Start: hcl.Pos{Byte: 7, Line: 1, Column: 8},
   497  						End:   hcl.Pos{Byte: 7, Line: 1, Column: 8},
   498  					},
   499  				},
   500  			},
   501  		},
   502  		{
   503  			`"hello, \"world\"!"`,
   504  			[]Token{
   505  				{
   506  					Type:  TokenOQuote,
   507  					Bytes: []byte(`"`),
   508  					Range: hcl.Range{
   509  						Start: hcl.Pos{Byte: 0, Line: 1, Column: 1},
   510  						End:   hcl.Pos{Byte: 1, Line: 1, Column: 2},
   511  					},
   512  				},
   513  				{
   514  					Type:  TokenQuotedLit,
   515  					Bytes: []byte(`hello, \"world\"!`), // The escapes are handled by the parser, not the scanner
   516  					Range: hcl.Range{
   517  						Start: hcl.Pos{Byte: 1, Line: 1, Column: 2},
   518  						End:   hcl.Pos{Byte: 18, Line: 1, Column: 19},
   519  					},
   520  				},
   521  				{
   522  					Type:  TokenCQuote,
   523  					Bytes: []byte(`"`),
   524  					Range: hcl.Range{
   525  						Start: hcl.Pos{Byte: 18, Line: 1, Column: 19},
   526  						End:   hcl.Pos{Byte: 19, Line: 1, Column: 20},
   527  					},
   528  				},
   529  				{
   530  					Type:  TokenEOF,
   531  					Bytes: []byte{},
   532  					Range: hcl.Range{
   533  						Start: hcl.Pos{Byte: 19, Line: 1, Column: 20},
   534  						End:   hcl.Pos{Byte: 19, Line: 1, Column: 20},
   535  					},
   536  				},
   537  			},
   538  		},
   539  		{
   540  			`"hello $$"`,
   541  			[]Token{
   542  				{
   543  					Type:  TokenOQuote,
   544  					Bytes: []byte(`"`),
   545  					Range: hcl.Range{
   546  						Start: hcl.Pos{Byte: 0, Line: 1, Column: 1},
   547  						End:   hcl.Pos{Byte: 1, Line: 1, Column: 2},
   548  					},
   549  				},
   550  				{
   551  					Type:  TokenQuotedLit,
   552  					Bytes: []byte(`hello `),
   553  					Range: hcl.Range{
   554  						Start: hcl.Pos{Byte: 1, Line: 1, Column: 2},
   555  						End:   hcl.Pos{Byte: 7, Line: 1, Column: 8},
   556  					},
   557  				},
   558  				// This one scans a little oddly because of how the scanner
   559  				// handles the escaping of the dollar sign, but it's still
   560  				// good enough for the parser since it'll just concatenate
   561  				// these two string literals together anyway.
   562  				{
   563  					Type:  TokenQuotedLit,
   564  					Bytes: []byte(`$`),
   565  					Range: hcl.Range{
   566  						Start: hcl.Pos{Byte: 7, Line: 1, Column: 8},
   567  						End:   hcl.Pos{Byte: 8, Line: 1, Column: 9},
   568  					},
   569  				},
   570  				{
   571  					Type:  TokenQuotedLit,
   572  					Bytes: []byte(`$`),
   573  					Range: hcl.Range{
   574  						Start: hcl.Pos{Byte: 8, Line: 1, Column: 9},
   575  						End:   hcl.Pos{Byte: 9, Line: 1, Column: 10},
   576  					},
   577  				},
   578  				{
   579  					Type:  TokenCQuote,
   580  					Bytes: []byte(`"`),
   581  					Range: hcl.Range{
   582  						Start: hcl.Pos{Byte: 9, Line: 1, Column: 10},
   583  						End:   hcl.Pos{Byte: 10, Line: 1, Column: 11},
   584  					},
   585  				},
   586  				{
   587  					Type:  TokenEOF,
   588  					Bytes: []byte{},
   589  					Range: hcl.Range{
   590  						Start: hcl.Pos{Byte: 10, Line: 1, Column: 11},
   591  						End:   hcl.Pos{Byte: 10, Line: 1, Column: 11},
   592  					},
   593  				},
   594  			},
   595  		},
   596  		{
   597  			`"hello %%"`,
   598  			[]Token{
   599  				{
   600  					Type:  TokenOQuote,
   601  					Bytes: []byte(`"`),
   602  					Range: hcl.Range{
   603  						Start: hcl.Pos{Byte: 0, Line: 1, Column: 1},
   604  						End:   hcl.Pos{Byte: 1, Line: 1, Column: 2},
   605  					},
   606  				},
   607  				{
   608  					Type:  TokenQuotedLit,
   609  					Bytes: []byte(`hello `),
   610  					Range: hcl.Range{
   611  						Start: hcl.Pos{Byte: 1, Line: 1, Column: 2},
   612  						End:   hcl.Pos{Byte: 7, Line: 1, Column: 8},
   613  					},
   614  				},
   615  				// This one scans a little oddly because of how the scanner
   616  				// handles the escaping of the percent sign, but it's still
   617  				// good enough for the parser since it'll just concatenate
   618  				// these two string literals together anyway.
   619  				{
   620  					Type:  TokenQuotedLit,
   621  					Bytes: []byte(`%`),
   622  					Range: hcl.Range{
   623  						Start: hcl.Pos{Byte: 7, Line: 1, Column: 8},
   624  						End:   hcl.Pos{Byte: 8, Line: 1, Column: 9},
   625  					},
   626  				},
   627  				{
   628  					Type:  TokenQuotedLit,
   629  					Bytes: []byte(`%`),
   630  					Range: hcl.Range{
   631  						Start: hcl.Pos{Byte: 8, Line: 1, Column: 9},
   632  						End:   hcl.Pos{Byte: 9, Line: 1, Column: 10},
   633  					},
   634  				},
   635  				{
   636  					Type:  TokenCQuote,
   637  					Bytes: []byte(`"`),
   638  					Range: hcl.Range{
   639  						Start: hcl.Pos{Byte: 9, Line: 1, Column: 10},
   640  						End:   hcl.Pos{Byte: 10, Line: 1, Column: 11},
   641  					},
   642  				},
   643  				{
   644  					Type:  TokenEOF,
   645  					Bytes: []byte{},
   646  					Range: hcl.Range{
   647  						Start: hcl.Pos{Byte: 10, Line: 1, Column: 11},
   648  						End:   hcl.Pos{Byte: 10, Line: 1, Column: 11},
   649  					},
   650  				},
   651  			},
   652  		},
   653  		{
   654  			`"hello $"`,
   655  			[]Token{
   656  				{
   657  					Type:  TokenOQuote,
   658  					Bytes: []byte(`"`),
   659  					Range: hcl.Range{
   660  						Start: hcl.Pos{Byte: 0, Line: 1, Column: 1},
   661  						End:   hcl.Pos{Byte: 1, Line: 1, Column: 2},
   662  					},
   663  				},
   664  				{
   665  					Type:  TokenQuotedLit,
   666  					Bytes: []byte(`hello `),
   667  					Range: hcl.Range{
   668  						Start: hcl.Pos{Byte: 1, Line: 1, Column: 2},
   669  						End:   hcl.Pos{Byte: 7, Line: 1, Column: 8},
   670  					},
   671  				},
   672  				{
   673  					Type:  TokenQuotedLit,
   674  					Bytes: []byte(`$`),
   675  					Range: hcl.Range{
   676  						Start: hcl.Pos{Byte: 7, Line: 1, Column: 8},
   677  						End:   hcl.Pos{Byte: 8, Line: 1, Column: 9},
   678  					},
   679  				},
   680  				{
   681  					Type:  TokenCQuote,
   682  					Bytes: []byte(`"`),
   683  					Range: hcl.Range{
   684  						Start: hcl.Pos{Byte: 8, Line: 1, Column: 9},
   685  						End:   hcl.Pos{Byte: 9, Line: 1, Column: 10},
   686  					},
   687  				},
   688  				{
   689  					Type:  TokenEOF,
   690  					Bytes: []byte{},
   691  					Range: hcl.Range{
   692  						Start: hcl.Pos{Byte: 9, Line: 1, Column: 10},
   693  						End:   hcl.Pos{Byte: 9, Line: 1, Column: 10},
   694  					},
   695  				},
   696  			},
   697  		},
   698  		{
   699  			`"hello %"`,
   700  			[]Token{
   701  				{
   702  					Type:  TokenOQuote,
   703  					Bytes: []byte(`"`),
   704  					Range: hcl.Range{
   705  						Start: hcl.Pos{Byte: 0, Line: 1, Column: 1},
   706  						End:   hcl.Pos{Byte: 1, Line: 1, Column: 2},
   707  					},
   708  				},
   709  				{
   710  					Type:  TokenQuotedLit,
   711  					Bytes: []byte(`hello `),
   712  					Range: hcl.Range{
   713  						Start: hcl.Pos{Byte: 1, Line: 1, Column: 2},
   714  						End:   hcl.Pos{Byte: 7, Line: 1, Column: 8},
   715  					},
   716  				},
   717  				{
   718  					Type:  TokenQuotedLit,
   719  					Bytes: []byte(`%`),
   720  					Range: hcl.Range{
   721  						Start: hcl.Pos{Byte: 7, Line: 1, Column: 8},
   722  						End:   hcl.Pos{Byte: 8, Line: 1, Column: 9},
   723  					},
   724  				},
   725  				{
   726  					Type:  TokenCQuote,
   727  					Bytes: []byte(`"`),
   728  					Range: hcl.Range{
   729  						Start: hcl.Pos{Byte: 8, Line: 1, Column: 9},
   730  						End:   hcl.Pos{Byte: 9, Line: 1, Column: 10},
   731  					},
   732  				},
   733  				{
   734  					Type:  TokenEOF,
   735  					Bytes: []byte{},
   736  					Range: hcl.Range{
   737  						Start: hcl.Pos{Byte: 9, Line: 1, Column: 10},
   738  						End:   hcl.Pos{Byte: 9, Line: 1, Column: 10},
   739  					},
   740  				},
   741  			},
   742  		},
   743  		{
   744  			`"hello $${world}"`,
   745  			[]Token{
   746  				{
   747  					Type:  TokenOQuote,
   748  					Bytes: []byte(`"`),
   749  					Range: hcl.Range{
   750  						Start: hcl.Pos{Byte: 0, Line: 1, Column: 1},
   751  						End:   hcl.Pos{Byte: 1, Line: 1, Column: 2},
   752  					},
   753  				},
   754  				{
   755  					Type:  TokenQuotedLit,
   756  					Bytes: []byte(`hello `),
   757  					Range: hcl.Range{
   758  						Start: hcl.Pos{Byte: 1, Line: 1, Column: 2},
   759  						End:   hcl.Pos{Byte: 7, Line: 1, Column: 8},
   760  					},
   761  				},
   762  				{
   763  					Type:  TokenQuotedLit,
   764  					Bytes: []byte(`$${`),
   765  					Range: hcl.Range{
   766  						Start: hcl.Pos{Byte: 7, Line: 1, Column: 8},
   767  						End:   hcl.Pos{Byte: 10, Line: 1, Column: 11},
   768  					},
   769  				},
   770  				{
   771  					Type:  TokenQuotedLit,
   772  					Bytes: []byte(`world}`),
   773  					Range: hcl.Range{
   774  						Start: hcl.Pos{Byte: 10, Line: 1, Column: 11},
   775  						End:   hcl.Pos{Byte: 16, Line: 1, Column: 17},
   776  					},
   777  				},
   778  				{
   779  					Type:  TokenCQuote,
   780  					Bytes: []byte(`"`),
   781  					Range: hcl.Range{
   782  						Start: hcl.Pos{Byte: 16, Line: 1, Column: 17},
   783  						End:   hcl.Pos{Byte: 17, Line: 1, Column: 18},
   784  					},
   785  				},
   786  				{
   787  					Type:  TokenEOF,
   788  					Bytes: []byte{},
   789  					Range: hcl.Range{
   790  						Start: hcl.Pos{Byte: 17, Line: 1, Column: 18},
   791  						End:   hcl.Pos{Byte: 17, Line: 1, Column: 18},
   792  					},
   793  				},
   794  			},
   795  		},
   796  		{
   797  			`"hello %%{world}"`,
   798  			[]Token{
   799  				{
   800  					Type:  TokenOQuote,
   801  					Bytes: []byte(`"`),
   802  					Range: hcl.Range{
   803  						Start: hcl.Pos{Byte: 0, Line: 1, Column: 1},
   804  						End:   hcl.Pos{Byte: 1, Line: 1, Column: 2},
   805  					},
   806  				},
   807  				{
   808  					Type:  TokenQuotedLit,
   809  					Bytes: []byte(`hello `),
   810  					Range: hcl.Range{
   811  						Start: hcl.Pos{Byte: 1, Line: 1, Column: 2},
   812  						End:   hcl.Pos{Byte: 7, Line: 1, Column: 8},
   813  					},
   814  				},
   815  				{
   816  					Type:  TokenQuotedLit,
   817  					Bytes: []byte(`%%{`),
   818  					Range: hcl.Range{
   819  						Start: hcl.Pos{Byte: 7, Line: 1, Column: 8},
   820  						End:   hcl.Pos{Byte: 10, Line: 1, Column: 11},
   821  					},
   822  				},
   823  				{
   824  					Type:  TokenQuotedLit,
   825  					Bytes: []byte(`world}`),
   826  					Range: hcl.Range{
   827  						Start: hcl.Pos{Byte: 10, Line: 1, Column: 11},
   828  						End:   hcl.Pos{Byte: 16, Line: 1, Column: 17},
   829  					},
   830  				},
   831  				{
   832  					Type:  TokenCQuote,
   833  					Bytes: []byte(`"`),
   834  					Range: hcl.Range{
   835  						Start: hcl.Pos{Byte: 16, Line: 1, Column: 17},
   836  						End:   hcl.Pos{Byte: 17, Line: 1, Column: 18},
   837  					},
   838  				},
   839  				{
   840  					Type:  TokenEOF,
   841  					Bytes: []byte{},
   842  					Range: hcl.Range{
   843  						Start: hcl.Pos{Byte: 17, Line: 1, Column: 18},
   844  						End:   hcl.Pos{Byte: 17, Line: 1, Column: 18},
   845  					},
   846  				},
   847  			},
   848  		},
   849  		{
   850  			`"hello %${world}"`,
   851  			[]Token{
   852  				{
   853  					Type:  TokenOQuote,
   854  					Bytes: []byte(`"`),
   855  					Range: hcl.Range{
   856  						Start: hcl.Pos{Byte: 0, Line: 1, Column: 1},
   857  						End:   hcl.Pos{Byte: 1, Line: 1, Column: 2},
   858  					},
   859  				},
   860  				{
   861  					Type:  TokenQuotedLit,
   862  					Bytes: []byte(`hello `),
   863  					Range: hcl.Range{
   864  						Start: hcl.Pos{Byte: 1, Line: 1, Column: 2},
   865  						End:   hcl.Pos{Byte: 7, Line: 1, Column: 8},
   866  					},
   867  				},
   868  				{
   869  					Type:  TokenQuotedLit,
   870  					Bytes: []byte(`%`),
   871  					Range: hcl.Range{
   872  						Start: hcl.Pos{Byte: 7, Line: 1, Column: 8},
   873  						End:   hcl.Pos{Byte: 8, Line: 1, Column: 9},
   874  					},
   875  				},
   876  				{
   877  					Type:  TokenTemplateInterp,
   878  					Bytes: []byte(`${`),
   879  					Range: hcl.Range{
   880  						Start: hcl.Pos{Byte: 8, Line: 1, Column: 9},
   881  						End:   hcl.Pos{Byte: 10, Line: 1, Column: 11},
   882  					},
   883  				},
   884  				{
   885  					Type:  TokenIdent,
   886  					Bytes: []byte(`world`),
   887  					Range: hcl.Range{
   888  						Start: hcl.Pos{Byte: 10, Line: 1, Column: 11},
   889  						End:   hcl.Pos{Byte: 15, Line: 1, Column: 16},
   890  					},
   891  				},
   892  				{
   893  					Type:  TokenTemplateSeqEnd,
   894  					Bytes: []byte(`}`),
   895  					Range: hcl.Range{
   896  						Start: hcl.Pos{Byte: 15, Line: 1, Column: 16},
   897  						End:   hcl.Pos{Byte: 16, Line: 1, Column: 17},
   898  					},
   899  				},
   900  				{
   901  					Type:  TokenCQuote,
   902  					Bytes: []byte(`"`),
   903  					Range: hcl.Range{
   904  						Start: hcl.Pos{Byte: 16, Line: 1, Column: 17},
   905  						End:   hcl.Pos{Byte: 17, Line: 1, Column: 18},
   906  					},
   907  				},
   908  				{
   909  					Type:  TokenEOF,
   910  					Bytes: []byte{},
   911  					Range: hcl.Range{
   912  						Start: hcl.Pos{Byte: 17, Line: 1, Column: 18},
   913  						End:   hcl.Pos{Byte: 17, Line: 1, Column: 18},
   914  					},
   915  				},
   916  			},
   917  		},
   918  
   919  		// Templates with interpolations and control sequences
   920  		{
   921  			`"${1}"`,
   922  			[]Token{
   923  				{
   924  					Type:  TokenOQuote,
   925  					Bytes: []byte(`"`),
   926  					Range: hcl.Range{
   927  						Start: hcl.Pos{Byte: 0, Line: 1, Column: 1},
   928  						End:   hcl.Pos{Byte: 1, Line: 1, Column: 2},
   929  					},
   930  				},
   931  				{
   932  					Type:  TokenTemplateInterp,
   933  					Bytes: []byte(`${`),
   934  					Range: hcl.Range{
   935  						Start: hcl.Pos{Byte: 1, Line: 1, Column: 2},
   936  						End:   hcl.Pos{Byte: 3, Line: 1, Column: 4},
   937  					},
   938  				},
   939  				{
   940  					Type:  TokenNumberLit,
   941  					Bytes: []byte(`1`),
   942  					Range: hcl.Range{
   943  						Start: hcl.Pos{Byte: 3, Line: 1, Column: 4},
   944  						End:   hcl.Pos{Byte: 4, Line: 1, Column: 5},
   945  					},
   946  				},
   947  				{
   948  					Type:  TokenTemplateSeqEnd,
   949  					Bytes: []byte(`}`),
   950  					Range: hcl.Range{
   951  						Start: hcl.Pos{Byte: 4, Line: 1, Column: 5},
   952  						End:   hcl.Pos{Byte: 5, Line: 1, Column: 6},
   953  					},
   954  				},
   955  				{
   956  					Type:  TokenCQuote,
   957  					Bytes: []byte(`"`),
   958  					Range: hcl.Range{
   959  						Start: hcl.Pos{Byte: 5, Line: 1, Column: 6},
   960  						End:   hcl.Pos{Byte: 6, Line: 1, Column: 7},
   961  					},
   962  				},
   963  				{
   964  					Type:  TokenEOF,
   965  					Bytes: []byte{},
   966  					Range: hcl.Range{
   967  						Start: hcl.Pos{Byte: 6, Line: 1, Column: 7},
   968  						End:   hcl.Pos{Byte: 6, Line: 1, Column: 7},
   969  					},
   970  				},
   971  			},
   972  		},
   973  		{
   974  			`"%{a}"`,
   975  			[]Token{
   976  				{
   977  					Type:  TokenOQuote,
   978  					Bytes: []byte(`"`),
   979  					Range: hcl.Range{
   980  						Start: hcl.Pos{Byte: 0, Line: 1, Column: 1},
   981  						End:   hcl.Pos{Byte: 1, Line: 1, Column: 2},
   982  					},
   983  				},
   984  				{
   985  					Type:  TokenTemplateControl,
   986  					Bytes: []byte(`%{`),
   987  					Range: hcl.Range{
   988  						Start: hcl.Pos{Byte: 1, Line: 1, Column: 2},
   989  						End:   hcl.Pos{Byte: 3, Line: 1, Column: 4},
   990  					},
   991  				},
   992  				{
   993  					Type:  TokenIdent,
   994  					Bytes: []byte(`a`),
   995  					Range: hcl.Range{
   996  						Start: hcl.Pos{Byte: 3, Line: 1, Column: 4},
   997  						End:   hcl.Pos{Byte: 4, Line: 1, Column: 5},
   998  					},
   999  				},
  1000  				{
  1001  					Type:  TokenTemplateSeqEnd,
  1002  					Bytes: []byte(`}`),
  1003  					Range: hcl.Range{
  1004  						Start: hcl.Pos{Byte: 4, Line: 1, Column: 5},
  1005  						End:   hcl.Pos{Byte: 5, Line: 1, Column: 6},
  1006  					},
  1007  				},
  1008  				{
  1009  					Type:  TokenCQuote,
  1010  					Bytes: []byte(`"`),
  1011  					Range: hcl.Range{
  1012  						Start: hcl.Pos{Byte: 5, Line: 1, Column: 6},
  1013  						End:   hcl.Pos{Byte: 6, Line: 1, Column: 7},
  1014  					},
  1015  				},
  1016  				{
  1017  					Type:  TokenEOF,
  1018  					Bytes: []byte{},
  1019  					Range: hcl.Range{
  1020  						Start: hcl.Pos{Byte: 6, Line: 1, Column: 7},
  1021  						End:   hcl.Pos{Byte: 6, Line: 1, Column: 7},
  1022  					},
  1023  				},
  1024  			},
  1025  		},
  1026  		{
  1027  			`"${{}}"`,
  1028  			[]Token{
  1029  				{
  1030  					Type:  TokenOQuote,
  1031  					Bytes: []byte(`"`),
  1032  					Range: hcl.Range{
  1033  						Start: hcl.Pos{Byte: 0, Line: 1, Column: 1},
  1034  						End:   hcl.Pos{Byte: 1, Line: 1, Column: 2},
  1035  					},
  1036  				},
  1037  				{
  1038  					Type:  TokenTemplateInterp,
  1039  					Bytes: []byte(`${`),
  1040  					Range: hcl.Range{
  1041  						Start: hcl.Pos{Byte: 1, Line: 1, Column: 2},
  1042  						End:   hcl.Pos{Byte: 3, Line: 1, Column: 4},
  1043  					},
  1044  				},
  1045  				{
  1046  					Type:  TokenOBrace,
  1047  					Bytes: []byte(`{`),
  1048  					Range: hcl.Range{
  1049  						Start: hcl.Pos{Byte: 3, Line: 1, Column: 4},
  1050  						End:   hcl.Pos{Byte: 4, Line: 1, Column: 5},
  1051  					},
  1052  				},
  1053  				{
  1054  					Type:  TokenCBrace,
  1055  					Bytes: []byte(`}`),
  1056  					Range: hcl.Range{
  1057  						Start: hcl.Pos{Byte: 4, Line: 1, Column: 5},
  1058  						End:   hcl.Pos{Byte: 5, Line: 1, Column: 6},
  1059  					},
  1060  				},
  1061  				{
  1062  					Type:  TokenTemplateSeqEnd,
  1063  					Bytes: []byte(`}`),
  1064  					Range: hcl.Range{
  1065  						Start: hcl.Pos{Byte: 5, Line: 1, Column: 6},
  1066  						End:   hcl.Pos{Byte: 6, Line: 1, Column: 7},
  1067  					},
  1068  				},
  1069  				{
  1070  					Type:  TokenCQuote,
  1071  					Bytes: []byte(`"`),
  1072  					Range: hcl.Range{
  1073  						Start: hcl.Pos{Byte: 6, Line: 1, Column: 7},
  1074  						End:   hcl.Pos{Byte: 7, Line: 1, Column: 8},
  1075  					},
  1076  				},
  1077  				{
  1078  					Type:  TokenEOF,
  1079  					Bytes: []byte{},
  1080  					Range: hcl.Range{
  1081  						Start: hcl.Pos{Byte: 7, Line: 1, Column: 8},
  1082  						End:   hcl.Pos{Byte: 7, Line: 1, Column: 8},
  1083  					},
  1084  				},
  1085  			},
  1086  		},
  1087  		{
  1088  			`"${""}"`,
  1089  			[]Token{
  1090  				{
  1091  					Type:  TokenOQuote,
  1092  					Bytes: []byte(`"`),
  1093  					Range: hcl.Range{
  1094  						Start: hcl.Pos{Byte: 0, Line: 1, Column: 1},
  1095  						End:   hcl.Pos{Byte: 1, Line: 1, Column: 2},
  1096  					},
  1097  				},
  1098  				{
  1099  					Type:  TokenTemplateInterp,
  1100  					Bytes: []byte(`${`),
  1101  					Range: hcl.Range{
  1102  						Start: hcl.Pos{Byte: 1, Line: 1, Column: 2},
  1103  						End:   hcl.Pos{Byte: 3, Line: 1, Column: 4},
  1104  					},
  1105  				},
  1106  				{
  1107  					Type:  TokenOQuote,
  1108  					Bytes: []byte(`"`),
  1109  					Range: hcl.Range{
  1110  						Start: hcl.Pos{Byte: 3, Line: 1, Column: 4},
  1111  						End:   hcl.Pos{Byte: 4, Line: 1, Column: 5},
  1112  					},
  1113  				},
  1114  				{
  1115  					Type:  TokenCQuote,
  1116  					Bytes: []byte(`"`),
  1117  					Range: hcl.Range{
  1118  						Start: hcl.Pos{Byte: 4, Line: 1, Column: 5},
  1119  						End:   hcl.Pos{Byte: 5, Line: 1, Column: 6},
  1120  					},
  1121  				},
  1122  				{
  1123  					Type:  TokenTemplateSeqEnd,
  1124  					Bytes: []byte(`}`),
  1125  					Range: hcl.Range{
  1126  						Start: hcl.Pos{Byte: 5, Line: 1, Column: 6},
  1127  						End:   hcl.Pos{Byte: 6, Line: 1, Column: 7},
  1128  					},
  1129  				},
  1130  				{
  1131  					Type:  TokenCQuote,
  1132  					Bytes: []byte(`"`),
  1133  					Range: hcl.Range{
  1134  						Start: hcl.Pos{Byte: 6, Line: 1, Column: 7},
  1135  						End:   hcl.Pos{Byte: 7, Line: 1, Column: 8},
  1136  					},
  1137  				},
  1138  				{
  1139  					Type:  TokenEOF,
  1140  					Bytes: []byte{},
  1141  					Range: hcl.Range{
  1142  						Start: hcl.Pos{Byte: 7, Line: 1, Column: 8},
  1143  						End:   hcl.Pos{Byte: 7, Line: 1, Column: 8},
  1144  					},
  1145  				},
  1146  			},
  1147  		},
  1148  		{
  1149  			`"${"${a}"}"`,
  1150  			[]Token{
  1151  				{
  1152  					Type:  TokenOQuote,
  1153  					Bytes: []byte(`"`),
  1154  					Range: hcl.Range{
  1155  						Start: hcl.Pos{Byte: 0, Line: 1, Column: 1},
  1156  						End:   hcl.Pos{Byte: 1, Line: 1, Column: 2},
  1157  					},
  1158  				},
  1159  				{
  1160  					Type:  TokenTemplateInterp,
  1161  					Bytes: []byte(`${`),
  1162  					Range: hcl.Range{
  1163  						Start: hcl.Pos{Byte: 1, Line: 1, Column: 2},
  1164  						End:   hcl.Pos{Byte: 3, Line: 1, Column: 4},
  1165  					},
  1166  				},
  1167  				{
  1168  					Type:  TokenOQuote,
  1169  					Bytes: []byte(`"`),
  1170  					Range: hcl.Range{
  1171  						Start: hcl.Pos{Byte: 3, Line: 1, Column: 4},
  1172  						End:   hcl.Pos{Byte: 4, Line: 1, Column: 5},
  1173  					},
  1174  				},
  1175  				{
  1176  					Type:  TokenTemplateInterp,
  1177  					Bytes: []byte(`${`),
  1178  					Range: hcl.Range{
  1179  						Start: hcl.Pos{Byte: 4, Line: 1, Column: 5},
  1180  						End:   hcl.Pos{Byte: 6, Line: 1, Column: 7},
  1181  					},
  1182  				},
  1183  				{
  1184  					Type:  TokenIdent,
  1185  					Bytes: []byte(`a`),
  1186  					Range: hcl.Range{
  1187  						Start: hcl.Pos{Byte: 6, Line: 1, Column: 7},
  1188  						End:   hcl.Pos{Byte: 7, Line: 1, Column: 8},
  1189  					},
  1190  				},
  1191  				{
  1192  					Type:  TokenTemplateSeqEnd,
  1193  					Bytes: []byte(`}`),
  1194  					Range: hcl.Range{
  1195  						Start: hcl.Pos{Byte: 7, Line: 1, Column: 8},
  1196  						End:   hcl.Pos{Byte: 8, Line: 1, Column: 9},
  1197  					},
  1198  				},
  1199  				{
  1200  					Type:  TokenCQuote,
  1201  					Bytes: []byte(`"`),
  1202  					Range: hcl.Range{
  1203  						Start: hcl.Pos{Byte: 8, Line: 1, Column: 9},
  1204  						End:   hcl.Pos{Byte: 9, Line: 1, Column: 10},
  1205  					},
  1206  				},
  1207  				{
  1208  					Type:  TokenTemplateSeqEnd,
  1209  					Bytes: []byte(`}`),
  1210  					Range: hcl.Range{
  1211  						Start: hcl.Pos{Byte: 9, Line: 1, Column: 10},
  1212  						End:   hcl.Pos{Byte: 10, Line: 1, Column: 11},
  1213  					},
  1214  				},
  1215  				{
  1216  					Type:  TokenCQuote,
  1217  					Bytes: []byte(`"`),
  1218  					Range: hcl.Range{
  1219  						Start: hcl.Pos{Byte: 10, Line: 1, Column: 11},
  1220  						End:   hcl.Pos{Byte: 11, Line: 1, Column: 12},
  1221  					},
  1222  				},
  1223  				{
  1224  					Type:  TokenEOF,
  1225  					Bytes: []byte{},
  1226  					Range: hcl.Range{
  1227  						Start: hcl.Pos{Byte: 11, Line: 1, Column: 12},
  1228  						End:   hcl.Pos{Byte: 11, Line: 1, Column: 12},
  1229  					},
  1230  				},
  1231  			},
  1232  		},
  1233  		{
  1234  			`"${"${a} foo"}"`,
  1235  			[]Token{
  1236  				{
  1237  					Type:  TokenOQuote,
  1238  					Bytes: []byte(`"`),
  1239  					Range: hcl.Range{
  1240  						Start: hcl.Pos{Byte: 0, Line: 1, Column: 1},
  1241  						End:   hcl.Pos{Byte: 1, Line: 1, Column: 2},
  1242  					},
  1243  				},
  1244  				{
  1245  					Type:  TokenTemplateInterp,
  1246  					Bytes: []byte(`${`),
  1247  					Range: hcl.Range{
  1248  						Start: hcl.Pos{Byte: 1, Line: 1, Column: 2},
  1249  						End:   hcl.Pos{Byte: 3, Line: 1, Column: 4},
  1250  					},
  1251  				},
  1252  				{
  1253  					Type:  TokenOQuote,
  1254  					Bytes: []byte(`"`),
  1255  					Range: hcl.Range{
  1256  						Start: hcl.Pos{Byte: 3, Line: 1, Column: 4},
  1257  						End:   hcl.Pos{Byte: 4, Line: 1, Column: 5},
  1258  					},
  1259  				},
  1260  				{
  1261  					Type:  TokenTemplateInterp,
  1262  					Bytes: []byte(`${`),
  1263  					Range: hcl.Range{
  1264  						Start: hcl.Pos{Byte: 4, Line: 1, Column: 5},
  1265  						End:   hcl.Pos{Byte: 6, Line: 1, Column: 7},
  1266  					},
  1267  				},
  1268  				{
  1269  					Type:  TokenIdent,
  1270  					Bytes: []byte(`a`),
  1271  					Range: hcl.Range{
  1272  						Start: hcl.Pos{Byte: 6, Line: 1, Column: 7},
  1273  						End:   hcl.Pos{Byte: 7, Line: 1, Column: 8},
  1274  					},
  1275  				},
  1276  				{
  1277  					Type:  TokenTemplateSeqEnd,
  1278  					Bytes: []byte(`}`),
  1279  					Range: hcl.Range{
  1280  						Start: hcl.Pos{Byte: 7, Line: 1, Column: 8},
  1281  						End:   hcl.Pos{Byte: 8, Line: 1, Column: 9},
  1282  					},
  1283  				},
  1284  				{
  1285  					Type:  TokenQuotedLit,
  1286  					Bytes: []byte(` foo`),
  1287  					Range: hcl.Range{
  1288  						Start: hcl.Pos{Byte: 8, Line: 1, Column: 9},
  1289  						End:   hcl.Pos{Byte: 12, Line: 1, Column: 13},
  1290  					},
  1291  				},
  1292  				{
  1293  					Type:  TokenCQuote,
  1294  					Bytes: []byte(`"`),
  1295  					Range: hcl.Range{
  1296  						Start: hcl.Pos{Byte: 12, Line: 1, Column: 13},
  1297  						End:   hcl.Pos{Byte: 13, Line: 1, Column: 14},
  1298  					},
  1299  				},
  1300  				{
  1301  					Type:  TokenTemplateSeqEnd,
  1302  					Bytes: []byte(`}`),
  1303  					Range: hcl.Range{
  1304  						Start: hcl.Pos{Byte: 13, Line: 1, Column: 14},
  1305  						End:   hcl.Pos{Byte: 14, Line: 1, Column: 15},
  1306  					},
  1307  				},
  1308  				{
  1309  					Type:  TokenCQuote,
  1310  					Bytes: []byte(`"`),
  1311  					Range: hcl.Range{
  1312  						Start: hcl.Pos{Byte: 14, Line: 1, Column: 15},
  1313  						End:   hcl.Pos{Byte: 15, Line: 1, Column: 16},
  1314  					},
  1315  				},
  1316  				{
  1317  					Type:  TokenEOF,
  1318  					Bytes: []byte{},
  1319  					Range: hcl.Range{
  1320  						Start: hcl.Pos{Byte: 15, Line: 1, Column: 16},
  1321  						End:   hcl.Pos{Byte: 15, Line: 1, Column: 16},
  1322  					},
  1323  				},
  1324  			},
  1325  		},
  1326  
  1327  		// Heredoc Templates
  1328  		{
  1329  			`<<EOT
  1330  hello world
  1331  EOT
  1332  `,
  1333  			[]Token{
  1334  				{
  1335  					Type:  TokenOHeredoc,
  1336  					Bytes: []byte("<<EOT\n"),
  1337  					Range: hcl.Range{
  1338  						Start: hcl.Pos{Byte: 0, Line: 1, Column: 1},
  1339  						End:   hcl.Pos{Byte: 6, Line: 2, Column: 1},
  1340  					},
  1341  				},
  1342  				{
  1343  					Type:  TokenStringLit,
  1344  					Bytes: []byte("hello world\n"),
  1345  					Range: hcl.Range{
  1346  						Start: hcl.Pos{Byte: 6, Line: 2, Column: 1},
  1347  						End:   hcl.Pos{Byte: 18, Line: 3, Column: 1},
  1348  					},
  1349  				},
  1350  				{
  1351  					Type:  TokenCHeredoc,
  1352  					Bytes: []byte("EOT"),
  1353  					Range: hcl.Range{
  1354  						Start: hcl.Pos{Byte: 18, Line: 3, Column: 1},
  1355  						End:   hcl.Pos{Byte: 21, Line: 3, Column: 4},
  1356  					},
  1357  				},
  1358  				{
  1359  					Type:  TokenNewline,
  1360  					Bytes: []byte("\n"),
  1361  					Range: hcl.Range{
  1362  						Start: hcl.Pos{Byte: 21, Line: 3, Column: 4},
  1363  						End:   hcl.Pos{Byte: 22, Line: 4, Column: 1},
  1364  					},
  1365  				},
  1366  				{
  1367  					Type:  TokenEOF,
  1368  					Bytes: []byte{},
  1369  					Range: hcl.Range{
  1370  						Start: hcl.Pos{Byte: 22, Line: 4, Column: 1},
  1371  						End:   hcl.Pos{Byte: 22, Line: 4, Column: 1},
  1372  					},
  1373  				},
  1374  			},
  1375  		},
  1376  		{
  1377  			"<<EOT\r\nhello world\r\nEOT\r\n", // intentional windows-style line endings
  1378  			[]Token{
  1379  				{
  1380  					Type:  TokenOHeredoc,
  1381  					Bytes: []byte("<<EOT\r\n"),
  1382  					Range: hcl.Range{
  1383  						Start: hcl.Pos{Byte: 0, Line: 1, Column: 1},
  1384  						End:   hcl.Pos{Byte: 7, Line: 2, Column: 1},
  1385  					},
  1386  				},
  1387  				{
  1388  					Type:  TokenStringLit,
  1389  					Bytes: []byte("hello world\r\n"),
  1390  					Range: hcl.Range{
  1391  						Start: hcl.Pos{Byte: 7, Line: 2, Column: 1},
  1392  						End:   hcl.Pos{Byte: 20, Line: 3, Column: 1},
  1393  					},
  1394  				},
  1395  				{
  1396  					Type:  TokenCHeredoc,
  1397  					Bytes: []byte("EOT"),
  1398  					Range: hcl.Range{
  1399  						Start: hcl.Pos{Byte: 20, Line: 3, Column: 1},
  1400  						End:   hcl.Pos{Byte: 23, Line: 3, Column: 4},
  1401  					},
  1402  				},
  1403  				{
  1404  					Type:  TokenNewline,
  1405  					Bytes: []byte("\r\n"),
  1406  					Range: hcl.Range{
  1407  						Start: hcl.Pos{Byte: 23, Line: 3, Column: 4},
  1408  						End:   hcl.Pos{Byte: 25, Line: 4, Column: 1},
  1409  					},
  1410  				},
  1411  				{
  1412  					Type:  TokenEOF,
  1413  					Bytes: []byte{},
  1414  					Range: hcl.Range{
  1415  						Start: hcl.Pos{Byte: 25, Line: 4, Column: 1},
  1416  						End:   hcl.Pos{Byte: 25, Line: 4, Column: 1},
  1417  					},
  1418  				},
  1419  			},
  1420  		},
  1421  		{
  1422  			`<<EOT
  1423  hello ${name}
  1424  EOT
  1425  `,
  1426  			[]Token{
  1427  				{
  1428  					Type:  TokenOHeredoc,
  1429  					Bytes: []byte("<<EOT\n"),
  1430  					Range: hcl.Range{
  1431  						Start: hcl.Pos{Byte: 0, Line: 1, Column: 1},
  1432  						End:   hcl.Pos{Byte: 6, Line: 2, Column: 1},
  1433  					},
  1434  				},
  1435  				{
  1436  					Type:  TokenStringLit,
  1437  					Bytes: []byte("hello "),
  1438  					Range: hcl.Range{
  1439  						Start: hcl.Pos{Byte: 6, Line: 2, Column: 1},
  1440  						End:   hcl.Pos{Byte: 12, Line: 2, Column: 7},
  1441  					},
  1442  				},
  1443  				{
  1444  					Type:  TokenTemplateInterp,
  1445  					Bytes: []byte("${"),
  1446  					Range: hcl.Range{
  1447  						Start: hcl.Pos{Byte: 12, Line: 2, Column: 7},
  1448  						End:   hcl.Pos{Byte: 14, Line: 2, Column: 9},
  1449  					},
  1450  				},
  1451  				{
  1452  					Type:  TokenIdent,
  1453  					Bytes: []byte("name"),
  1454  					Range: hcl.Range{
  1455  						Start: hcl.Pos{Byte: 14, Line: 2, Column: 9},
  1456  						End:   hcl.Pos{Byte: 18, Line: 2, Column: 13},
  1457  					},
  1458  				},
  1459  				{
  1460  					Type:  TokenTemplateSeqEnd,
  1461  					Bytes: []byte("}"),
  1462  					Range: hcl.Range{
  1463  						Start: hcl.Pos{Byte: 18, Line: 2, Column: 13},
  1464  						End:   hcl.Pos{Byte: 19, Line: 2, Column: 14},
  1465  					},
  1466  				},
  1467  				{
  1468  					Type:  TokenStringLit,
  1469  					Bytes: []byte("\n"),
  1470  					Range: hcl.Range{
  1471  						Start: hcl.Pos{Byte: 19, Line: 2, Column: 14},
  1472  						End:   hcl.Pos{Byte: 20, Line: 3, Column: 1},
  1473  					},
  1474  				},
  1475  				{
  1476  					Type:  TokenCHeredoc,
  1477  					Bytes: []byte("EOT"),
  1478  					Range: hcl.Range{
  1479  						Start: hcl.Pos{Byte: 20, Line: 3, Column: 1},
  1480  						End:   hcl.Pos{Byte: 23, Line: 3, Column: 4},
  1481  					},
  1482  				},
  1483  				{
  1484  					Type:  TokenNewline,
  1485  					Bytes: []byte("\n"),
  1486  					Range: hcl.Range{
  1487  						Start: hcl.Pos{Byte: 23, Line: 3, Column: 4},
  1488  						End:   hcl.Pos{Byte: 24, Line: 4, Column: 1},
  1489  					},
  1490  				},
  1491  				{
  1492  					Type:  TokenEOF,
  1493  					Bytes: []byte{},
  1494  					Range: hcl.Range{
  1495  						Start: hcl.Pos{Byte: 24, Line: 4, Column: 1},
  1496  						End:   hcl.Pos{Byte: 24, Line: 4, Column: 1},
  1497  					},
  1498  				},
  1499  			},
  1500  		},
  1501  		{
  1502  			`<<EOT
  1503  ${name}EOT
  1504  EOT
  1505  `,
  1506  			[]Token{
  1507  				{
  1508  					Type:  TokenOHeredoc,
  1509  					Bytes: []byte("<<EOT\n"),
  1510  					Range: hcl.Range{
  1511  						Start: hcl.Pos{Byte: 0, Line: 1, Column: 1},
  1512  						End:   hcl.Pos{Byte: 6, Line: 2, Column: 1},
  1513  					},
  1514  				},
  1515  				{
  1516  					Type:  TokenTemplateInterp,
  1517  					Bytes: []byte("${"),
  1518  					Range: hcl.Range{
  1519  						Start: hcl.Pos{Byte: 6, Line: 2, Column: 1},
  1520  						End:   hcl.Pos{Byte: 8, Line: 2, Column: 3},
  1521  					},
  1522  				},
  1523  				{
  1524  					Type:  TokenIdent,
  1525  					Bytes: []byte("name"),
  1526  					Range: hcl.Range{
  1527  						Start: hcl.Pos{Byte: 8, Line: 2, Column: 3},
  1528  						End:   hcl.Pos{Byte: 12, Line: 2, Column: 7},
  1529  					},
  1530  				},
  1531  				{
  1532  					Type:  TokenTemplateSeqEnd,
  1533  					Bytes: []byte("}"),
  1534  					Range: hcl.Range{
  1535  						Start: hcl.Pos{Byte: 12, Line: 2, Column: 7},
  1536  						End:   hcl.Pos{Byte: 13, Line: 2, Column: 8},
  1537  					},
  1538  				},
  1539  				{
  1540  					Type:  TokenStringLit,
  1541  					Bytes: []byte("EOT\n"),
  1542  					Range: hcl.Range{
  1543  						Start: hcl.Pos{Byte: 13, Line: 2, Column: 8},
  1544  						End:   hcl.Pos{Byte: 17, Line: 3, Column: 1},
  1545  					},
  1546  				},
  1547  				{
  1548  					Type:  TokenCHeredoc,
  1549  					Bytes: []byte("EOT"),
  1550  					Range: hcl.Range{
  1551  						Start: hcl.Pos{Byte: 17, Line: 3, Column: 1},
  1552  						End:   hcl.Pos{Byte: 20, Line: 3, Column: 4},
  1553  					},
  1554  				},
  1555  				{
  1556  					Type:  TokenNewline,
  1557  					Bytes: []byte("\n"),
  1558  					Range: hcl.Range{
  1559  						Start: hcl.Pos{Byte: 20, Line: 3, Column: 4},
  1560  						End:   hcl.Pos{Byte: 21, Line: 4, Column: 1},
  1561  					},
  1562  				},
  1563  				{
  1564  					Type:  TokenEOF,
  1565  					Bytes: []byte{},
  1566  					Range: hcl.Range{
  1567  						Start: hcl.Pos{Byte: 21, Line: 4, Column: 1},
  1568  						End:   hcl.Pos{Byte: 21, Line: 4, Column: 1},
  1569  					},
  1570  				},
  1571  			},
  1572  		},
  1573  		{
  1574  			`<<EOT
  1575    hello world
  1576  EOT
  1577  `,
  1578  			[]Token{
  1579  				{
  1580  					Type:  TokenOHeredoc,
  1581  					Bytes: []byte("<<EOT\n"),
  1582  					Range: hcl.Range{
  1583  						Start: hcl.Pos{Byte: 0, Line: 1, Column: 1},
  1584  						End:   hcl.Pos{Byte: 6, Line: 2, Column: 1},
  1585  					},
  1586  				},
  1587  				{
  1588  					Type:  TokenStringLit,
  1589  					Bytes: []byte("  hello world\n"),
  1590  					Range: hcl.Range{
  1591  						Start: hcl.Pos{Byte: 6, Line: 2, Column: 1},
  1592  						End:   hcl.Pos{Byte: 20, Line: 3, Column: 1},
  1593  					},
  1594  				},
  1595  				{
  1596  					Type:  TokenCHeredoc,
  1597  					Bytes: []byte("EOT"),
  1598  					Range: hcl.Range{
  1599  						Start: hcl.Pos{Byte: 20, Line: 3, Column: 1},
  1600  						End:   hcl.Pos{Byte: 23, Line: 3, Column: 4},
  1601  					},
  1602  				},
  1603  				{
  1604  					Type:  TokenNewline,
  1605  					Bytes: []byte("\n"),
  1606  					Range: hcl.Range{
  1607  						Start: hcl.Pos{Byte: 23, Line: 3, Column: 4},
  1608  						End:   hcl.Pos{Byte: 24, Line: 4, Column: 1},
  1609  					},
  1610  				},
  1611  				{
  1612  					Type:  TokenEOF,
  1613  					Bytes: []byte{},
  1614  					Range: hcl.Range{
  1615  						Start: hcl.Pos{Byte: 24, Line: 4, Column: 1},
  1616  						End:   hcl.Pos{Byte: 24, Line: 4, Column: 1},
  1617  					},
  1618  				},
  1619  			},
  1620  		},
  1621  		{
  1622  			`<<-EOT
  1623    hello world
  1624  EOT
  1625  `,
  1626  			[]Token{
  1627  				{
  1628  					Type:  TokenOHeredoc,
  1629  					Bytes: []byte("<<-EOT\n"),
  1630  					Range: hcl.Range{
  1631  						Start: hcl.Pos{Byte: 0, Line: 1, Column: 1},
  1632  						End:   hcl.Pos{Byte: 7, Line: 2, Column: 1},
  1633  					},
  1634  				},
  1635  				{
  1636  					Type:  TokenStringLit,
  1637  					Bytes: []byte("  hello world\n"),
  1638  					Range: hcl.Range{
  1639  						Start: hcl.Pos{Byte: 7, Line: 2, Column: 1},
  1640  						End:   hcl.Pos{Byte: 21, Line: 3, Column: 1},
  1641  					},
  1642  				},
  1643  				{
  1644  					Type:  TokenCHeredoc,
  1645  					Bytes: []byte("EOT"),
  1646  					Range: hcl.Range{
  1647  						Start: hcl.Pos{Byte: 21, Line: 3, Column: 1},
  1648  						End:   hcl.Pos{Byte: 24, Line: 3, Column: 4},
  1649  					},
  1650  				},
  1651  				{
  1652  					Type:  TokenNewline,
  1653  					Bytes: []byte("\n"),
  1654  					Range: hcl.Range{
  1655  						Start: hcl.Pos{Byte: 24, Line: 3, Column: 4},
  1656  						End:   hcl.Pos{Byte: 25, Line: 4, Column: 1},
  1657  					},
  1658  				},
  1659  				{
  1660  					Type:  TokenEOF,
  1661  					Bytes: []byte{},
  1662  					Range: hcl.Range{
  1663  						Start: hcl.Pos{Byte: 25, Line: 4, Column: 1},
  1664  						End:   hcl.Pos{Byte: 25, Line: 4, Column: 1},
  1665  					},
  1666  				},
  1667  			},
  1668  		},
  1669  		{
  1670  			`<<-EOT
  1671    hello world
  1672   EOT
  1673  `,
  1674  			[]Token{
  1675  				{
  1676  					Type:  TokenOHeredoc,
  1677  					Bytes: []byte("<<-EOT\n"),
  1678  					Range: hcl.Range{
  1679  						Start: hcl.Pos{Byte: 0, Line: 1, Column: 1},
  1680  						End:   hcl.Pos{Byte: 7, Line: 2, Column: 1},
  1681  					},
  1682  				},
  1683  				{
  1684  					Type:  TokenStringLit,
  1685  					Bytes: []byte("  hello world\n"),
  1686  					Range: hcl.Range{
  1687  						Start: hcl.Pos{Byte: 7, Line: 2, Column: 1},
  1688  						End:   hcl.Pos{Byte: 21, Line: 3, Column: 1},
  1689  					},
  1690  				},
  1691  				{
  1692  					Type:  TokenCHeredoc,
  1693  					Bytes: []byte(" EOT"),
  1694  					Range: hcl.Range{
  1695  						Start: hcl.Pos{Byte: 21, Line: 3, Column: 1},
  1696  						End:   hcl.Pos{Byte: 25, Line: 3, Column: 5},
  1697  					},
  1698  				},
  1699  				{
  1700  					Type:  TokenNewline,
  1701  					Bytes: []byte("\n"),
  1702  					Range: hcl.Range{
  1703  						Start: hcl.Pos{Byte: 25, Line: 3, Column: 5},
  1704  						End:   hcl.Pos{Byte: 26, Line: 4, Column: 1},
  1705  					},
  1706  				},
  1707  				{
  1708  					Type:  TokenEOF,
  1709  					Bytes: []byte{},
  1710  					Range: hcl.Range{
  1711  						Start: hcl.Pos{Byte: 26, Line: 4, Column: 1},
  1712  						End:   hcl.Pos{Byte: 26, Line: 4, Column: 1},
  1713  					},
  1714  				},
  1715  			},
  1716  		},
  1717  		{
  1718  			`<<EOF
  1719  ${<<-EOF
  1720  hello
  1721  EOF
  1722  }
  1723  EOF
  1724  `,
  1725  			[]Token{
  1726  				{
  1727  					Type:  TokenOHeredoc,
  1728  					Bytes: []byte("<<EOF\n"),
  1729  					Range: hcl.Range{
  1730  						Start: hcl.Pos{Byte: 0, Line: 1, Column: 1},
  1731  						End:   hcl.Pos{Byte: 6, Line: 2, Column: 1},
  1732  					},
  1733  				},
  1734  				{
  1735  					Type:  TokenTemplateInterp,
  1736  					Bytes: []byte("${"),
  1737  					Range: hcl.Range{
  1738  						Start: hcl.Pos{Byte: 6, Line: 2, Column: 1},
  1739  						End:   hcl.Pos{Byte: 8, Line: 2, Column: 3},
  1740  					},
  1741  				},
  1742  				{
  1743  					Type:  TokenOHeredoc,
  1744  					Bytes: []byte("<<-EOF\n"),
  1745  					Range: hcl.Range{
  1746  						Start: hcl.Pos{Byte: 8, Line: 2, Column: 3},
  1747  						End:   hcl.Pos{Byte: 15, Line: 3, Column: 1},
  1748  					},
  1749  				},
  1750  				{
  1751  					Type:  TokenStringLit,
  1752  					Bytes: []byte("hello\n"),
  1753  					Range: hcl.Range{
  1754  						Start: hcl.Pos{Byte: 15, Line: 3, Column: 1},
  1755  						End:   hcl.Pos{Byte: 21, Line: 4, Column: 1},
  1756  					},
  1757  				},
  1758  				{
  1759  					Type:  TokenCHeredoc,
  1760  					Bytes: []byte("EOF"),
  1761  					Range: hcl.Range{
  1762  						Start: hcl.Pos{Byte: 21, Line: 4, Column: 1},
  1763  						End:   hcl.Pos{Byte: 24, Line: 4, Column: 4},
  1764  					},
  1765  				},
  1766  				{
  1767  					Type:  TokenNewline,
  1768  					Bytes: []byte("\n"),
  1769  					Range: hcl.Range{
  1770  						Start: hcl.Pos{Byte: 24, Line: 4, Column: 4},
  1771  						End:   hcl.Pos{Byte: 25, Line: 5, Column: 1},
  1772  					},
  1773  				},
  1774  				{
  1775  					Type:  TokenTemplateSeqEnd,
  1776  					Bytes: []byte("}"),
  1777  					Range: hcl.Range{
  1778  						Start: hcl.Pos{Byte: 25, Line: 5, Column: 1},
  1779  						End:   hcl.Pos{Byte: 26, Line: 5, Column: 2},
  1780  					},
  1781  				},
  1782  				{
  1783  					Type:  TokenStringLit,
  1784  					Bytes: []byte("\n"),
  1785  					Range: hcl.Range{
  1786  						Start: hcl.Pos{Byte: 26, Line: 5, Column: 2},
  1787  						End:   hcl.Pos{Byte: 27, Line: 6, Column: 1},
  1788  					},
  1789  				},
  1790  				{
  1791  					Type:  TokenCHeredoc,
  1792  					Bytes: []byte("EOF"),
  1793  					Range: hcl.Range{
  1794  						Start: hcl.Pos{Byte: 27, Line: 6, Column: 1},
  1795  						End:   hcl.Pos{Byte: 30, Line: 6, Column: 4},
  1796  					},
  1797  				},
  1798  				{
  1799  					Type:  TokenNewline,
  1800  					Bytes: []byte("\n"),
  1801  					Range: hcl.Range{
  1802  						Start: hcl.Pos{Byte: 30, Line: 6, Column: 4},
  1803  						End:   hcl.Pos{Byte: 31, Line: 7, Column: 1},
  1804  					},
  1805  				},
  1806  				{
  1807  					Type:  TokenEOF,
  1808  					Bytes: []byte{},
  1809  					Range: hcl.Range{
  1810  						Start: hcl.Pos{Byte: 31, Line: 7, Column: 1},
  1811  						End:   hcl.Pos{Byte: 31, Line: 7, Column: 1},
  1812  					},
  1813  				},
  1814  			},
  1815  		},
  1816  		{
  1817  			`<<EOF 
  1818  hello
  1819  EOF
  1820  `,
  1821  			// `EOF ` is not a valid identifier
  1822  			// so `<<EOF ` is not a valid TokenOHeredoc
  1823  			[]Token{
  1824  				{
  1825  					Type:  TokenLessThan,
  1826  					Bytes: []byte("<"),
  1827  					Range: hcl.Range{
  1828  						Start: hcl.Pos{Byte: 0, Line: 1, Column: 1},
  1829  						End:   hcl.Pos{Byte: 1, Line: 1, Column: 2},
  1830  					},
  1831  				},
  1832  				{
  1833  					Type:  TokenLessThan,
  1834  					Bytes: []byte("<"),
  1835  					Range: hcl.Range{
  1836  						Start: hcl.Pos{Byte: 1, Line: 1, Column: 2},
  1837  						End:   hcl.Pos{Byte: 2, Line: 1, Column: 3},
  1838  					},
  1839  				},
  1840  				{
  1841  					Type:  TokenIdent,
  1842  					Bytes: []byte("EOF"),
  1843  					Range: hcl.Range{
  1844  						Start: hcl.Pos{Byte: 2, Line: 1, Column: 3},
  1845  						End:   hcl.Pos{Byte: 5, Line: 1, Column: 6},
  1846  					},
  1847  				},
  1848  				{
  1849  					Type:  TokenNewline,
  1850  					Bytes: []byte("\n"),
  1851  					Range: hcl.Range{
  1852  						Start: hcl.Pos{Byte: 6, Line: 1, Column: 7},
  1853  						End:   hcl.Pos{Byte: 7, Line: 2, Column: 1},
  1854  					},
  1855  				},
  1856  				{
  1857  					Type:  TokenIdent,
  1858  					Bytes: []byte("hello"),
  1859  					Range: hcl.Range{
  1860  						Start: hcl.Pos{Byte: 7, Line: 2, Column: 1},
  1861  						End:   hcl.Pos{Byte: 12, Line: 2, Column: 6},
  1862  					},
  1863  				},
  1864  				{
  1865  					Type:  TokenNewline,
  1866  					Bytes: []byte("\n"),
  1867  					Range: hcl.Range{
  1868  						Start: hcl.Pos{Byte: 12, Line: 2, Column: 6},
  1869  						End:   hcl.Pos{Byte: 13, Line: 3, Column: 1},
  1870  					},
  1871  				},
  1872  				{
  1873  					Type:  TokenIdent,
  1874  					Bytes: []byte("EOF"),
  1875  					Range: hcl.Range{
  1876  						Start: hcl.Pos{Byte: 13, Line: 3, Column: 1},
  1877  						End:   hcl.Pos{Byte: 16, Line: 3, Column: 4},
  1878  					},
  1879  				},
  1880  
  1881  				{
  1882  					Type:  TokenNewline,
  1883  					Bytes: []byte("\n"),
  1884  					Range: hcl.Range{
  1885  						Start: hcl.Pos{Byte: 16, Line: 3, Column: 4},
  1886  						End:   hcl.Pos{Byte: 17, Line: 4, Column: 1},
  1887  					},
  1888  				},
  1889  				{
  1890  					Type:  TokenEOF,
  1891  					Bytes: []byte{},
  1892  					Range: hcl.Range{
  1893  						Start: hcl.Pos{Byte: 17, Line: 4, Column: 1},
  1894  						End:   hcl.Pos{Byte: 17, Line: 4, Column: 1},
  1895  					},
  1896  				},
  1897  			},
  1898  		},
  1899  		{
  1900  			`<<EOF 
  1901  hello
  1902  EOF 
  1903  `,
  1904  			// `EOF ` is not a valid identifier
  1905  			// so `<<EOF ` is not a valid TokenOHeredoc
  1906  			[]Token{
  1907  				{
  1908  					Type:  TokenLessThan,
  1909  					Bytes: []byte("<"),
  1910  					Range: hcl.Range{
  1911  						Start: hcl.Pos{Byte: 0, Line: 1, Column: 1},
  1912  						End:   hcl.Pos{Byte: 1, Line: 1, Column: 2},
  1913  					},
  1914  				},
  1915  				{
  1916  					Type:  TokenLessThan,
  1917  					Bytes: []byte("<"),
  1918  					Range: hcl.Range{
  1919  						Start: hcl.Pos{Byte: 1, Line: 1, Column: 2},
  1920  						End:   hcl.Pos{Byte: 2, Line: 1, Column: 3},
  1921  					},
  1922  				},
  1923  				{
  1924  					Type:  TokenIdent,
  1925  					Bytes: []byte("EOF"),
  1926  					Range: hcl.Range{
  1927  						Start: hcl.Pos{Byte: 2, Line: 1, Column: 3},
  1928  						End:   hcl.Pos{Byte: 5, Line: 1, Column: 6},
  1929  					},
  1930  				},
  1931  				{
  1932  					Type:  TokenNewline,
  1933  					Bytes: []byte("\n"),
  1934  					Range: hcl.Range{
  1935  						Start: hcl.Pos{Byte: 6, Line: 1, Column: 7},
  1936  						End:   hcl.Pos{Byte: 7, Line: 2, Column: 1},
  1937  					},
  1938  				},
  1939  				{
  1940  					Type:  TokenIdent,
  1941  					Bytes: []byte("hello"),
  1942  					Range: hcl.Range{
  1943  						Start: hcl.Pos{Byte: 7, Line: 2, Column: 1},
  1944  						End:   hcl.Pos{Byte: 12, Line: 2, Column: 6},
  1945  					},
  1946  				},
  1947  				{
  1948  					Type:  TokenNewline,
  1949  					Bytes: []byte("\n"),
  1950  					Range: hcl.Range{
  1951  						Start: hcl.Pos{Byte: 12, Line: 2, Column: 6},
  1952  						End:   hcl.Pos{Byte: 13, Line: 3, Column: 1},
  1953  					},
  1954  				},
  1955  				{
  1956  					Type:  TokenIdent,
  1957  					Bytes: []byte("EOF"),
  1958  					Range: hcl.Range{
  1959  						Start: hcl.Pos{Byte: 13, Line: 3, Column: 1},
  1960  						End:   hcl.Pos{Byte: 16, Line: 3, Column: 4},
  1961  					},
  1962  				},
  1963  
  1964  				{
  1965  					Type:  TokenNewline,
  1966  					Bytes: []byte("\n"),
  1967  					Range: hcl.Range{
  1968  						Start: hcl.Pos{Byte: 17, Line: 3, Column: 5},
  1969  						End:   hcl.Pos{Byte: 18, Line: 4, Column: 1},
  1970  					},
  1971  				},
  1972  				{
  1973  					Type:  TokenEOF,
  1974  					Bytes: []byte{},
  1975  					Range: hcl.Range{
  1976  						Start: hcl.Pos{Byte: 18, Line: 4, Column: 1},
  1977  						End:   hcl.Pos{Byte: 18, Line: 4, Column: 1},
  1978  					},
  1979  				},
  1980  			},
  1981  		},
  1982  
  1983  		// Combinations
  1984  		{
  1985  			` (1 + 2) * 3 `,
  1986  			[]Token{
  1987  				{
  1988  					Type:  TokenOParen,
  1989  					Bytes: []byte(`(`),
  1990  					Range: hcl.Range{
  1991  						Start: hcl.Pos{Byte: 1, Line: 1, Column: 2},
  1992  						End:   hcl.Pos{Byte: 2, Line: 1, Column: 3},
  1993  					},
  1994  				},
  1995  				{
  1996  					Type:  TokenNumberLit,
  1997  					Bytes: []byte(`1`),
  1998  					Range: hcl.Range{
  1999  						Start: hcl.Pos{Byte: 2, Line: 1, Column: 3},
  2000  						End:   hcl.Pos{Byte: 3, Line: 1, Column: 4},
  2001  					},
  2002  				},
  2003  				{
  2004  					Type:  TokenPlus,
  2005  					Bytes: []byte(`+`),
  2006  					Range: hcl.Range{
  2007  						Start: hcl.Pos{Byte: 4, Line: 1, Column: 5},
  2008  						End:   hcl.Pos{Byte: 5, Line: 1, Column: 6},
  2009  					},
  2010  				},
  2011  				{
  2012  					Type:  TokenNumberLit,
  2013  					Bytes: []byte(`2`),
  2014  					Range: hcl.Range{
  2015  						Start: hcl.Pos{Byte: 6, Line: 1, Column: 7},
  2016  						End:   hcl.Pos{Byte: 7, Line: 1, Column: 8},
  2017  					},
  2018  				},
  2019  				{
  2020  					Type:  TokenCParen,
  2021  					Bytes: []byte(`)`),
  2022  					Range: hcl.Range{
  2023  						Start: hcl.Pos{Byte: 7, Line: 1, Column: 8},
  2024  						End:   hcl.Pos{Byte: 8, Line: 1, Column: 9},
  2025  					},
  2026  				},
  2027  				{
  2028  					Type:  TokenStar,
  2029  					Bytes: []byte(`*`),
  2030  					Range: hcl.Range{
  2031  						Start: hcl.Pos{Byte: 9, Line: 1, Column: 10},
  2032  						End:   hcl.Pos{Byte: 10, Line: 1, Column: 11},
  2033  					},
  2034  				},
  2035  				{
  2036  					Type:  TokenNumberLit,
  2037  					Bytes: []byte(`3`),
  2038  					Range: hcl.Range{
  2039  						Start: hcl.Pos{Byte: 11, Line: 1, Column: 12},
  2040  						End:   hcl.Pos{Byte: 12, Line: 1, Column: 13},
  2041  					},
  2042  				},
  2043  				{
  2044  					Type:  TokenEOF,
  2045  					Bytes: []byte{},
  2046  					Range: hcl.Range{
  2047  						Start: hcl.Pos{Byte: 13, Line: 1, Column: 14},
  2048  						End:   hcl.Pos{Byte: 13, Line: 1, Column: 14},
  2049  					},
  2050  				},
  2051  			},
  2052  		},
  2053  		{
  2054  			`9%8`,
  2055  			[]Token{
  2056  				{
  2057  					Type:  TokenNumberLit,
  2058  					Bytes: []byte(`9`),
  2059  					Range: hcl.Range{
  2060  						Start: hcl.Pos{Byte: 0, Line: 1, Column: 1},
  2061  						End:   hcl.Pos{Byte: 1, Line: 1, Column: 2},
  2062  					},
  2063  				},
  2064  				{
  2065  					Type:  TokenPercent,
  2066  					Bytes: []byte(`%`),
  2067  					Range: hcl.Range{
  2068  						Start: hcl.Pos{Byte: 1, Line: 1, Column: 2},
  2069  						End:   hcl.Pos{Byte: 2, Line: 1, Column: 3},
  2070  					},
  2071  				},
  2072  				{
  2073  					Type:  TokenNumberLit,
  2074  					Bytes: []byte(`8`),
  2075  					Range: hcl.Range{
  2076  						Start: hcl.Pos{Byte: 2, Line: 1, Column: 3},
  2077  						End:   hcl.Pos{Byte: 3, Line: 1, Column: 4},
  2078  					},
  2079  				},
  2080  				{
  2081  					Type:  TokenEOF,
  2082  					Bytes: []byte(``),
  2083  					Range: hcl.Range{
  2084  						Start: hcl.Pos{Byte: 3, Line: 1, Column: 4},
  2085  						End:   hcl.Pos{Byte: 3, Line: 1, Column: 4},
  2086  					},
  2087  				},
  2088  			},
  2089  		},
  2090  		{
  2091  			"\na = 1\n",
  2092  			[]Token{
  2093  				{
  2094  					Type:  TokenNewline,
  2095  					Bytes: []byte("\n"),
  2096  					Range: hcl.Range{
  2097  						Start: hcl.Pos{Byte: 0, Line: 1, Column: 1},
  2098  						End:   hcl.Pos{Byte: 1, Line: 2, Column: 1},
  2099  					},
  2100  				},
  2101  				{
  2102  					Type:  TokenIdent,
  2103  					Bytes: []byte("a"),
  2104  					Range: hcl.Range{
  2105  						Start: hcl.Pos{Byte: 1, Line: 2, Column: 1},
  2106  						End:   hcl.Pos{Byte: 2, Line: 2, Column: 2},
  2107  					},
  2108  				},
  2109  				{
  2110  					Type:  TokenEqual,
  2111  					Bytes: []byte("="),
  2112  					Range: hcl.Range{
  2113  						Start: hcl.Pos{Byte: 3, Line: 2, Column: 3},
  2114  						End:   hcl.Pos{Byte: 4, Line: 2, Column: 4},
  2115  					},
  2116  				},
  2117  				{
  2118  					Type:  TokenNumberLit,
  2119  					Bytes: []byte("1"),
  2120  					Range: hcl.Range{
  2121  						Start: hcl.Pos{Byte: 5, Line: 2, Column: 5},
  2122  						End:   hcl.Pos{Byte: 6, Line: 2, Column: 6},
  2123  					},
  2124  				},
  2125  				{
  2126  					Type:  TokenNewline,
  2127  					Bytes: []byte("\n"),
  2128  					Range: hcl.Range{
  2129  						Start: hcl.Pos{Byte: 6, Line: 2, Column: 6},
  2130  						End:   hcl.Pos{Byte: 7, Line: 3, Column: 1},
  2131  					},
  2132  				},
  2133  				{
  2134  					Type:  TokenEOF,
  2135  					Bytes: []byte{},
  2136  					Range: hcl.Range{
  2137  						Start: hcl.Pos{Byte: 7, Line: 3, Column: 1},
  2138  						End:   hcl.Pos{Byte: 7, Line: 3, Column: 1},
  2139  					},
  2140  				},
  2141  			},
  2142  		},
  2143  
  2144  		// Comments
  2145  		{
  2146  			"# hello\n",
  2147  			[]Token{
  2148  				{
  2149  					Type:  TokenComment,
  2150  					Bytes: []byte("# hello\n"),
  2151  					Range: hcl.Range{
  2152  						Start: hcl.Pos{Byte: 0, Line: 1, Column: 1},
  2153  						End:   hcl.Pos{Byte: 8, Line: 2, Column: 1},
  2154  					},
  2155  				},
  2156  				{
  2157  					Type:  TokenEOF,
  2158  					Bytes: []byte{},
  2159  					Range: hcl.Range{
  2160  						Start: hcl.Pos{Byte: 8, Line: 2, Column: 1},
  2161  						End:   hcl.Pos{Byte: 8, Line: 2, Column: 1},
  2162  					},
  2163  				},
  2164  			},
  2165  		},
  2166  		{
  2167  			"// hello\n",
  2168  			[]Token{
  2169  				{
  2170  					Type:  TokenComment,
  2171  					Bytes: []byte("// hello\n"),
  2172  					Range: hcl.Range{
  2173  						Start: hcl.Pos{Byte: 0, Line: 1, Column: 1},
  2174  						End:   hcl.Pos{Byte: 9, Line: 2, Column: 1},
  2175  					},
  2176  				},
  2177  				{
  2178  					Type:  TokenEOF,
  2179  					Bytes: []byte{},
  2180  					Range: hcl.Range{
  2181  						Start: hcl.Pos{Byte: 9, Line: 2, Column: 1},
  2182  						End:   hcl.Pos{Byte: 9, Line: 2, Column: 1},
  2183  					},
  2184  				},
  2185  			},
  2186  		},
  2187  		{
  2188  			"// hello\n// hello",
  2189  			[]Token{
  2190  				{
  2191  					Type:  TokenComment,
  2192  					Bytes: []byte("// hello\n"),
  2193  					Range: hcl.Range{
  2194  						Start: hcl.Pos{Byte: 0, Line: 1, Column: 1},
  2195  						End:   hcl.Pos{Byte: 9, Line: 2, Column: 1},
  2196  					},
  2197  				},
  2198  				{
  2199  					Type:  TokenComment,
  2200  					Bytes: []byte("// hello"),
  2201  					Range: hcl.Range{
  2202  						Start: hcl.Pos{Byte: 9, Line: 2, Column: 1},
  2203  						End:   hcl.Pos{Byte: 17, Line: 2, Column: 9},
  2204  					},
  2205  				},
  2206  				{
  2207  					Type:  TokenEOF,
  2208  					Bytes: []byte{},
  2209  					Range: hcl.Range{
  2210  						Start: hcl.Pos{Byte: 17, Line: 2, Column: 9},
  2211  						End:   hcl.Pos{Byte: 17, Line: 2, Column: 9},
  2212  					},
  2213  				},
  2214  			},
  2215  		},
  2216  		{
  2217  			"// hello\nfoo\n// hello",
  2218  			[]Token{
  2219  				{
  2220  					Type:  TokenComment,
  2221  					Bytes: []byte("// hello\n"),
  2222  					Range: hcl.Range{
  2223  						Start: hcl.Pos{Byte: 0, Line: 1, Column: 1},
  2224  						End:   hcl.Pos{Byte: 9, Line: 2, Column: 1},
  2225  					},
  2226  				},
  2227  				{
  2228  					Type:  TokenIdent,
  2229  					Bytes: []byte("foo"),
  2230  					Range: hcl.Range{
  2231  						Start: hcl.Pos{Byte: 9, Line: 2, Column: 1},
  2232  						End:   hcl.Pos{Byte: 12, Line: 2, Column: 4},
  2233  					},
  2234  				},
  2235  				{
  2236  					Type:  TokenNewline,
  2237  					Bytes: []byte("\n"),
  2238  					Range: hcl.Range{
  2239  						Start: hcl.Pos{Byte: 12, Line: 2, Column: 4},
  2240  						End:   hcl.Pos{Byte: 13, Line: 3, Column: 1},
  2241  					},
  2242  				},
  2243  				{
  2244  					Type:  TokenComment,
  2245  					Bytes: []byte("// hello"),
  2246  					Range: hcl.Range{
  2247  						Start: hcl.Pos{Byte: 13, Line: 3, Column: 1},
  2248  						End:   hcl.Pos{Byte: 21, Line: 3, Column: 9},
  2249  					},
  2250  				},
  2251  				{
  2252  					Type:  TokenEOF,
  2253  					Bytes: []byte{},
  2254  					Range: hcl.Range{
  2255  						Start: hcl.Pos{Byte: 21, Line: 3, Column: 9},
  2256  						End:   hcl.Pos{Byte: 21, Line: 3, Column: 9},
  2257  					},
  2258  				},
  2259  			},
  2260  		},
  2261  		{
  2262  			"# hello\n# hello",
  2263  			[]Token{
  2264  				{
  2265  					Type:  TokenComment,
  2266  					Bytes: []byte("# hello\n"),
  2267  					Range: hcl.Range{
  2268  						Start: hcl.Pos{Byte: 0, Line: 1, Column: 1},
  2269  						End:   hcl.Pos{Byte: 8, Line: 2, Column: 1},
  2270  					},
  2271  				},
  2272  				{
  2273  					Type:  TokenComment,
  2274  					Bytes: []byte("# hello"),
  2275  					Range: hcl.Range{
  2276  						Start: hcl.Pos{Byte: 8, Line: 2, Column: 1},
  2277  						End:   hcl.Pos{Byte: 15, Line: 2, Column: 8},
  2278  					},
  2279  				},
  2280  				{
  2281  					Type:  TokenEOF,
  2282  					Bytes: []byte{},
  2283  					Range: hcl.Range{
  2284  						Start: hcl.Pos{Byte: 15, Line: 2, Column: 8},
  2285  						End:   hcl.Pos{Byte: 15, Line: 2, Column: 8},
  2286  					},
  2287  				},
  2288  			},
  2289  		},
  2290  		{
  2291  			"# hello\nfoo\n# hello",
  2292  			[]Token{
  2293  				{
  2294  					Type:  TokenComment,
  2295  					Bytes: []byte("# hello\n"),
  2296  					Range: hcl.Range{
  2297  						Start: hcl.Pos{Byte: 0, Line: 1, Column: 1},
  2298  						End:   hcl.Pos{Byte: 8, Line: 2, Column: 1},
  2299  					},
  2300  				},
  2301  				{
  2302  					Type:  TokenIdent,
  2303  					Bytes: []byte("foo"),
  2304  					Range: hcl.Range{
  2305  						Start: hcl.Pos{Byte: 8, Line: 2, Column: 1},
  2306  						End:   hcl.Pos{Byte: 11, Line: 2, Column: 4},
  2307  					},
  2308  				},
  2309  				{
  2310  					Type:  TokenNewline,
  2311  					Bytes: []byte("\n"),
  2312  					Range: hcl.Range{
  2313  						Start: hcl.Pos{Byte: 11, Line: 2, Column: 4},
  2314  						End:   hcl.Pos{Byte: 12, Line: 3, Column: 1},
  2315  					},
  2316  				},
  2317  				{
  2318  					Type:  TokenComment,
  2319  					Bytes: []byte("# hello"),
  2320  					Range: hcl.Range{
  2321  						Start: hcl.Pos{Byte: 12, Line: 3, Column: 1},
  2322  						End:   hcl.Pos{Byte: 19, Line: 3, Column: 8},
  2323  					},
  2324  				},
  2325  				{
  2326  					Type:  TokenEOF,
  2327  					Bytes: []byte{},
  2328  					Range: hcl.Range{
  2329  						Start: hcl.Pos{Byte: 19, Line: 3, Column: 8},
  2330  						End:   hcl.Pos{Byte: 19, Line: 3, Column: 8},
  2331  					},
  2332  				},
  2333  			},
  2334  		},
  2335  		{
  2336  			"/* hello */",
  2337  			[]Token{
  2338  				{
  2339  					Type:  TokenComment,
  2340  					Bytes: []byte("/* hello */"),
  2341  					Range: hcl.Range{
  2342  						Start: hcl.Pos{Byte: 0, Line: 1, Column: 1},
  2343  						End:   hcl.Pos{Byte: 11, Line: 1, Column: 12},
  2344  					},
  2345  				},
  2346  				{
  2347  					Type:  TokenEOF,
  2348  					Bytes: []byte{},
  2349  					Range: hcl.Range{
  2350  						Start: hcl.Pos{Byte: 11, Line: 1, Column: 12},
  2351  						End:   hcl.Pos{Byte: 11, Line: 1, Column: 12},
  2352  					},
  2353  				},
  2354  			},
  2355  		},
  2356  		{
  2357  			"/* hello */ howdy /* hey */",
  2358  			[]Token{
  2359  				{
  2360  					Type:  TokenComment,
  2361  					Bytes: []byte("/* hello */"),
  2362  					Range: hcl.Range{
  2363  						Start: hcl.Pos{Byte: 0, Line: 1, Column: 1},
  2364  						End:   hcl.Pos{Byte: 11, Line: 1, Column: 12},
  2365  					},
  2366  				},
  2367  				{
  2368  					Type:  TokenIdent,
  2369  					Bytes: []byte("howdy"),
  2370  					Range: hcl.Range{
  2371  						Start: hcl.Pos{Byte: 12, Line: 1, Column: 13},
  2372  						End:   hcl.Pos{Byte: 17, Line: 1, Column: 18},
  2373  					},
  2374  				},
  2375  				{
  2376  					Type:  TokenComment,
  2377  					Bytes: []byte("/* hey */"),
  2378  					Range: hcl.Range{
  2379  						Start: hcl.Pos{Byte: 18, Line: 1, Column: 19},
  2380  						End:   hcl.Pos{Byte: 27, Line: 1, Column: 28},
  2381  					},
  2382  				},
  2383  				{
  2384  					Type:  TokenEOF,
  2385  					Bytes: []byte{},
  2386  					Range: hcl.Range{
  2387  						Start: hcl.Pos{Byte: 27, Line: 1, Column: 28},
  2388  						End:   hcl.Pos{Byte: 27, Line: 1, Column: 28},
  2389  					},
  2390  				},
  2391  			},
  2392  		},
  2393  
  2394  		// Invalid things
  2395  		{
  2396  			`🌻`,
  2397  			[]Token{
  2398  				{
  2399  					Type:  TokenInvalid,
  2400  					Bytes: []byte(`🌻`),
  2401  					Range: hcl.Range{
  2402  						Start: hcl.Pos{Byte: 0, Line: 1, Column: 1},
  2403  						End:   hcl.Pos{Byte: 4, Line: 1, Column: 2},
  2404  					},
  2405  				},
  2406  				{
  2407  					Type:  TokenEOF,
  2408  					Bytes: []byte{},
  2409  					Range: hcl.Range{
  2410  						Start: hcl.Pos{Byte: 4, Line: 1, Column: 2},
  2411  						End:   hcl.Pos{Byte: 4, Line: 1, Column: 2},
  2412  					},
  2413  				},
  2414  			},
  2415  		},
  2416  		{
  2417  			`|`,
  2418  			[]Token{
  2419  				{
  2420  					Type:  TokenBitwiseOr,
  2421  					Bytes: []byte(`|`),
  2422  					Range: hcl.Range{
  2423  						Start: hcl.Pos{Byte: 0, Line: 1, Column: 1},
  2424  						End:   hcl.Pos{Byte: 1, Line: 1, Column: 2},
  2425  					},
  2426  				},
  2427  				{
  2428  					Type:  TokenEOF,
  2429  					Bytes: []byte{},
  2430  					Range: hcl.Range{
  2431  						Start: hcl.Pos{Byte: 1, Line: 1, Column: 2},
  2432  						End:   hcl.Pos{Byte: 1, Line: 1, Column: 2},
  2433  					},
  2434  				},
  2435  			},
  2436  		},
  2437  		{
  2438  			"\x80", // UTF-8 continuation without an introducer
  2439  			[]Token{
  2440  				{
  2441  					Type:  TokenBadUTF8,
  2442  					Bytes: []byte{0x80},
  2443  					Range: hcl.Range{
  2444  						Start: hcl.Pos{Byte: 0, Line: 1, Column: 1},
  2445  						End:   hcl.Pos{Byte: 1, Line: 1, Column: 2},
  2446  					},
  2447  				},
  2448  				{
  2449  					Type:  TokenEOF,
  2450  					Bytes: []byte{},
  2451  					Range: hcl.Range{
  2452  						Start: hcl.Pos{Byte: 1, Line: 1, Column: 2},
  2453  						End:   hcl.Pos{Byte: 1, Line: 1, Column: 2},
  2454  					},
  2455  				},
  2456  			},
  2457  		},
  2458  		{
  2459  			" \x80\x80", // UTF-8 continuation without an introducer
  2460  			[]Token{
  2461  				{
  2462  					Type:  TokenBadUTF8,
  2463  					Bytes: []byte{0x80},
  2464  					Range: hcl.Range{
  2465  						Start: hcl.Pos{Byte: 1, Line: 1, Column: 2},
  2466  						End:   hcl.Pos{Byte: 2, Line: 1, Column: 3},
  2467  					},
  2468  				},
  2469  				{
  2470  					Type:  TokenBadUTF8,
  2471  					Bytes: []byte{0x80},
  2472  					Range: hcl.Range{
  2473  						Start: hcl.Pos{Byte: 2, Line: 1, Column: 3},
  2474  						End:   hcl.Pos{Byte: 3, Line: 1, Column: 4},
  2475  					},
  2476  				},
  2477  				{
  2478  					Type:  TokenEOF,
  2479  					Bytes: []byte{},
  2480  					Range: hcl.Range{
  2481  						Start: hcl.Pos{Byte: 3, Line: 1, Column: 4},
  2482  						End:   hcl.Pos{Byte: 3, Line: 1, Column: 4},
  2483  					},
  2484  				},
  2485  			},
  2486  		},
  2487  		{
  2488  			"\t\t",
  2489  			[]Token{
  2490  				{
  2491  					Type:  TokenEOF,
  2492  					Bytes: []byte{},
  2493  					Range: hcl.Range{
  2494  						Start: hcl.Pos{Byte: 2, Line: 1, Column: 3},
  2495  						End:   hcl.Pos{Byte: 2, Line: 1, Column: 3},
  2496  					},
  2497  				},
  2498  			},
  2499  		},
  2500  
  2501  		// Misc combinations that have come up in bug reports, etc.
  2502  		{
  2503  			"locals {\n  is_percent = percent_sign == \"%\" ? true : false\n}\n",
  2504  			[]Token{
  2505  				{
  2506  					Type:  TokenIdent,
  2507  					Bytes: []byte(`locals`),
  2508  					Range: hcl.Range{
  2509  						Start: hcl.Pos{Byte: 0, Line: 1, Column: 1},
  2510  						End:   hcl.Pos{Byte: 6, Line: 1, Column: 7},
  2511  					},
  2512  				},
  2513  				{
  2514  					Type:  TokenOBrace,
  2515  					Bytes: []byte{'{'},
  2516  					Range: hcl.Range{
  2517  						Start: hcl.Pos{Byte: 7, Line: 1, Column: 8},
  2518  						End:   hcl.Pos{Byte: 8, Line: 1, Column: 9},
  2519  					},
  2520  				},
  2521  				{
  2522  					Type:  TokenNewline,
  2523  					Bytes: []byte{'\n'},
  2524  					Range: hcl.Range{
  2525  						Start: hcl.Pos{Byte: 8, Line: 1, Column: 9},
  2526  						End:   hcl.Pos{Byte: 9, Line: 2, Column: 1},
  2527  					},
  2528  				},
  2529  				{
  2530  					Type:  TokenIdent,
  2531  					Bytes: []byte(`is_percent`),
  2532  					Range: hcl.Range{
  2533  						Start: hcl.Pos{Byte: 11, Line: 2, Column: 3},
  2534  						End:   hcl.Pos{Byte: 21, Line: 2, Column: 13},
  2535  					},
  2536  				},
  2537  				{
  2538  					Type:  TokenEqual,
  2539  					Bytes: []byte(`=`),
  2540  					Range: hcl.Range{
  2541  						Start: hcl.Pos{Byte: 22, Line: 2, Column: 14},
  2542  						End:   hcl.Pos{Byte: 23, Line: 2, Column: 15},
  2543  					},
  2544  				},
  2545  				{
  2546  					Type:  TokenIdent,
  2547  					Bytes: []byte(`percent_sign`),
  2548  					Range: hcl.Range{
  2549  						Start: hcl.Pos{Byte: 24, Line: 2, Column: 16},
  2550  						End:   hcl.Pos{Byte: 36, Line: 2, Column: 28},
  2551  					},
  2552  				},
  2553  				{
  2554  					Type:  TokenEqualOp,
  2555  					Bytes: []byte(`==`),
  2556  					Range: hcl.Range{
  2557  						Start: hcl.Pos{Byte: 37, Line: 2, Column: 29},
  2558  						End:   hcl.Pos{Byte: 39, Line: 2, Column: 31},
  2559  					},
  2560  				},
  2561  				{
  2562  					Type:  TokenOQuote,
  2563  					Bytes: []byte(`"`),
  2564  					Range: hcl.Range{
  2565  						Start: hcl.Pos{Byte: 40, Line: 2, Column: 32},
  2566  						End:   hcl.Pos{Byte: 41, Line: 2, Column: 33},
  2567  					},
  2568  				},
  2569  				{
  2570  					Type:  TokenQuotedLit,
  2571  					Bytes: []byte(`%`),
  2572  					Range: hcl.Range{
  2573  						Start: hcl.Pos{Byte: 41, Line: 2, Column: 33},
  2574  						End:   hcl.Pos{Byte: 42, Line: 2, Column: 34},
  2575  					},
  2576  				},
  2577  				{
  2578  					Type:  TokenCQuote,
  2579  					Bytes: []byte(`"`),
  2580  					Range: hcl.Range{
  2581  						Start: hcl.Pos{Byte: 42, Line: 2, Column: 34},
  2582  						End:   hcl.Pos{Byte: 43, Line: 2, Column: 35},
  2583  					},
  2584  				},
  2585  				{
  2586  					Type:  TokenQuestion,
  2587  					Bytes: []byte(`?`),
  2588  					Range: hcl.Range{
  2589  						Start: hcl.Pos{Byte: 44, Line: 2, Column: 36},
  2590  						End:   hcl.Pos{Byte: 45, Line: 2, Column: 37},
  2591  					},
  2592  				},
  2593  				{
  2594  					Type:  TokenIdent,
  2595  					Bytes: []byte(`true`),
  2596  					Range: hcl.Range{
  2597  						Start: hcl.Pos{Byte: 46, Line: 2, Column: 38},
  2598  						End:   hcl.Pos{Byte: 50, Line: 2, Column: 42},
  2599  					},
  2600  				},
  2601  				{
  2602  					Type:  TokenColon,
  2603  					Bytes: []byte(`:`),
  2604  					Range: hcl.Range{
  2605  						Start: hcl.Pos{Byte: 51, Line: 2, Column: 43},
  2606  						End:   hcl.Pos{Byte: 52, Line: 2, Column: 44},
  2607  					},
  2608  				},
  2609  				{
  2610  					Type:  TokenIdent,
  2611  					Bytes: []byte(`false`),
  2612  					Range: hcl.Range{
  2613  						Start: hcl.Pos{Byte: 53, Line: 2, Column: 45},
  2614  						End:   hcl.Pos{Byte: 58, Line: 2, Column: 50},
  2615  					},
  2616  				},
  2617  				{
  2618  					Type:  TokenNewline,
  2619  					Bytes: []byte{'\n'},
  2620  					Range: hcl.Range{
  2621  						Start: hcl.Pos{Byte: 58, Line: 2, Column: 50},
  2622  						End:   hcl.Pos{Byte: 59, Line: 3, Column: 1},
  2623  					},
  2624  				},
  2625  				{
  2626  					Type:  TokenCBrace,
  2627  					Bytes: []byte{'}'},
  2628  					Range: hcl.Range{
  2629  						Start: hcl.Pos{Byte: 59, Line: 3, Column: 1},
  2630  						End:   hcl.Pos{Byte: 60, Line: 3, Column: 2},
  2631  					},
  2632  				},
  2633  				{
  2634  					Type:  TokenNewline,
  2635  					Bytes: []byte{'\n'},
  2636  					Range: hcl.Range{
  2637  						Start: hcl.Pos{Byte: 60, Line: 3, Column: 2},
  2638  						End:   hcl.Pos{Byte: 61, Line: 4, Column: 1},
  2639  					},
  2640  				},
  2641  				{
  2642  					Type:  TokenEOF,
  2643  					Bytes: []byte{},
  2644  					Range: hcl.Range{
  2645  						Start: hcl.Pos{Byte: 61, Line: 4, Column: 1},
  2646  						End:   hcl.Pos{Byte: 61, Line: 4, Column: 1},
  2647  					},
  2648  				},
  2649  			},
  2650  		},
  2651  	}
  2652  
  2653  	for _, test := range tests {
  2654  		t.Run(test.input, func(t *testing.T) {
  2655  			got := scanTokens([]byte(test.input), "", hcl.Pos{Byte: 0, Line: 1, Column: 1}, scanNormal)
  2656  
  2657  			if diff := cmp.Diff(test.want, got); diff != "" {
  2658  				t.Errorf("wrong result\n%s", diff)
  2659  			}
  2660  		})
  2661  	}
  2662  }
  2663  
  2664  func TestScanTokens_template(t *testing.T) {
  2665  	tests := []struct {
  2666  		input string
  2667  		want  []Token
  2668  	}{
  2669  		// Empty input
  2670  		{
  2671  			``,
  2672  			[]Token{
  2673  				{
  2674  					Type:  TokenEOF,
  2675  					Bytes: []byte{},
  2676  					Range: hcl.Range{
  2677  						Start: hcl.Pos{Byte: 0, Line: 1, Column: 1},
  2678  						End:   hcl.Pos{Byte: 0, Line: 1, Column: 1},
  2679  					},
  2680  				},
  2681  			},
  2682  		},
  2683  
  2684  		// Simple literals
  2685  		{
  2686  			` hello `,
  2687  			[]Token{
  2688  				{
  2689  					Type:  TokenStringLit,
  2690  					Bytes: []byte(` hello `),
  2691  					Range: hcl.Range{
  2692  						Start: hcl.Pos{Byte: 0, Line: 1, Column: 1},
  2693  						End:   hcl.Pos{Byte: 7, Line: 1, Column: 8},
  2694  					},
  2695  				},
  2696  				{
  2697  					Type:  TokenEOF,
  2698  					Bytes: []byte{},
  2699  					Range: hcl.Range{
  2700  						Start: hcl.Pos{Byte: 7, Line: 1, Column: 8},
  2701  						End:   hcl.Pos{Byte: 7, Line: 1, Column: 8},
  2702  					},
  2703  				},
  2704  			},
  2705  		},
  2706  		{
  2707  			"\nhello\n",
  2708  			[]Token{
  2709  				{
  2710  					Type:  TokenStringLit,
  2711  					Bytes: []byte("\n"),
  2712  					Range: hcl.Range{
  2713  						Start: hcl.Pos{Byte: 0, Line: 1, Column: 1},
  2714  						End:   hcl.Pos{Byte: 1, Line: 2, Column: 1},
  2715  					},
  2716  				},
  2717  				{
  2718  					Type:  TokenStringLit,
  2719  					Bytes: []byte("hello\n"),
  2720  					Range: hcl.Range{
  2721  						Start: hcl.Pos{Byte: 1, Line: 2, Column: 1},
  2722  						End:   hcl.Pos{Byte: 7, Line: 3, Column: 1},
  2723  					},
  2724  				},
  2725  				{
  2726  					Type:  TokenEOF,
  2727  					Bytes: []byte{},
  2728  					Range: hcl.Range{
  2729  						Start: hcl.Pos{Byte: 7, Line: 3, Column: 1},
  2730  						End:   hcl.Pos{Byte: 7, Line: 3, Column: 1},
  2731  					},
  2732  				},
  2733  			},
  2734  		},
  2735  		{
  2736  			"hello ${foo} hello",
  2737  			[]Token{
  2738  				{
  2739  					Type:  TokenStringLit,
  2740  					Bytes: []byte("hello "),
  2741  					Range: hcl.Range{
  2742  						Start: hcl.Pos{Byte: 0, Line: 1, Column: 1},
  2743  						End:   hcl.Pos{Byte: 6, Line: 1, Column: 7},
  2744  					},
  2745  				},
  2746  				{
  2747  					Type:  TokenTemplateInterp,
  2748  					Bytes: []byte("${"),
  2749  					Range: hcl.Range{
  2750  						Start: hcl.Pos{Byte: 6, Line: 1, Column: 7},
  2751  						End:   hcl.Pos{Byte: 8, Line: 1, Column: 9},
  2752  					},
  2753  				},
  2754  				{
  2755  					Type:  TokenIdent,
  2756  					Bytes: []byte("foo"),
  2757  					Range: hcl.Range{
  2758  						Start: hcl.Pos{Byte: 8, Line: 1, Column: 9},
  2759  						End:   hcl.Pos{Byte: 11, Line: 1, Column: 12},
  2760  					},
  2761  				},
  2762  				{
  2763  					Type:  TokenTemplateSeqEnd,
  2764  					Bytes: []byte("}"),
  2765  					Range: hcl.Range{
  2766  						Start: hcl.Pos{Byte: 11, Line: 1, Column: 12},
  2767  						End:   hcl.Pos{Byte: 12, Line: 1, Column: 13},
  2768  					},
  2769  				},
  2770  				{
  2771  					Type:  TokenStringLit,
  2772  					Bytes: []byte(" hello"),
  2773  					Range: hcl.Range{
  2774  						Start: hcl.Pos{Byte: 12, Line: 1, Column: 13},
  2775  						End:   hcl.Pos{Byte: 18, Line: 1, Column: 19},
  2776  					},
  2777  				},
  2778  				{
  2779  					Type:  TokenEOF,
  2780  					Bytes: []byte{},
  2781  					Range: hcl.Range{
  2782  						Start: hcl.Pos{Byte: 18, Line: 1, Column: 19},
  2783  						End:   hcl.Pos{Byte: 18, Line: 1, Column: 19},
  2784  					},
  2785  				},
  2786  			},
  2787  		},
  2788  		{
  2789  			"hello ${~foo~} hello",
  2790  			[]Token{
  2791  				{
  2792  					Type:  TokenStringLit,
  2793  					Bytes: []byte("hello "),
  2794  					Range: hcl.Range{
  2795  						Start: hcl.Pos{Byte: 0, Line: 1, Column: 1},
  2796  						End:   hcl.Pos{Byte: 6, Line: 1, Column: 7},
  2797  					},
  2798  				},
  2799  				{
  2800  					Type:  TokenTemplateInterp,
  2801  					Bytes: []byte("${~"),
  2802  					Range: hcl.Range{
  2803  						Start: hcl.Pos{Byte: 6, Line: 1, Column: 7},
  2804  						End:   hcl.Pos{Byte: 9, Line: 1, Column: 10},
  2805  					},
  2806  				},
  2807  				{
  2808  					Type:  TokenIdent,
  2809  					Bytes: []byte("foo"),
  2810  					Range: hcl.Range{
  2811  						Start: hcl.Pos{Byte: 9, Line: 1, Column: 10},
  2812  						End:   hcl.Pos{Byte: 12, Line: 1, Column: 13},
  2813  					},
  2814  				},
  2815  				{
  2816  					Type:  TokenTemplateSeqEnd,
  2817  					Bytes: []byte("~}"),
  2818  					Range: hcl.Range{
  2819  						Start: hcl.Pos{Byte: 12, Line: 1, Column: 13},
  2820  						End:   hcl.Pos{Byte: 14, Line: 1, Column: 15},
  2821  					},
  2822  				},
  2823  				{
  2824  					Type:  TokenStringLit,
  2825  					Bytes: []byte(" hello"),
  2826  					Range: hcl.Range{
  2827  						Start: hcl.Pos{Byte: 14, Line: 1, Column: 15},
  2828  						End:   hcl.Pos{Byte: 20, Line: 1, Column: 21},
  2829  					},
  2830  				},
  2831  				{
  2832  					Type:  TokenEOF,
  2833  					Bytes: []byte{},
  2834  					Range: hcl.Range{
  2835  						Start: hcl.Pos{Byte: 20, Line: 1, Column: 21},
  2836  						End:   hcl.Pos{Byte: 20, Line: 1, Column: 21},
  2837  					},
  2838  				},
  2839  			},
  2840  		},
  2841  	}
  2842  
  2843  	for _, test := range tests {
  2844  		t.Run(test.input, func(t *testing.T) {
  2845  			got := scanTokens([]byte(test.input), "", hcl.Pos{Byte: 0, Line: 1, Column: 1}, scanTemplate)
  2846  
  2847  			if diff := cmp.Diff(test.want, got); diff != "" {
  2848  				t.Errorf("wrong result\n%s", diff)
  2849  			}
  2850  		})
  2851  	}
  2852  }