github.com/tiagovtristao/plz@v13.4.0+incompatible/src/parse/asp/lexer_test.go (about)

     1  package asp
     2  
     3  import (
     4  	"strings"
     5  	"testing"
     6  
     7  	"github.com/stretchr/testify/assert"
     8  )
     9  
    10  func assertToken(t *testing.T, tok Token, tokenType rune, value string, line, column, offset int) {
    11  	assert.EqualValues(t, tokenType, tok.Type, "incorrect type")
    12  	assert.Equal(t, value, tok.Value, "incorrect value")
    13  	assert.Equal(t, line, tok.Pos.Line, "incorrect line")
    14  	assert.Equal(t, column, tok.Pos.Column, "incorrect column")
    15  	assert.Equal(t, offset, tok.Pos.Offset, "incorrect offset")
    16  }
    17  
    18  func TestLexBasic(t *testing.T) {
    19  	l := newLexer(strings.NewReader("hello world"))
    20  	assertToken(t, l.Next(), Ident, "hello", 1, 1, 1)
    21  	assertToken(t, l.Peek(), Ident, "world", 1, 7, 7)
    22  	assertToken(t, l.Next(), Ident, "world", 1, 7, 7)
    23  	assertToken(t, l.Next(), EOL, "", 1, 12, 12)
    24  	assertToken(t, l.Next(), EOF, "", 2, 1, 13)
    25  }
    26  
    27  func TestLexMultiline(t *testing.T) {
    28  	l := newLexer(strings.NewReader("hello\nworld\n"))
    29  	assertToken(t, l.Next(), Ident, "hello", 1, 1, 1)
    30  	assertToken(t, l.Next(), EOL, "", 1, 6, 6)
    31  	assertToken(t, l.Next(), Ident, "world", 2, 1, 7)
    32  	assertToken(t, l.Next(), EOL, "", 2, 6, 12)
    33  	assertToken(t, l.Next(), EOF, "", 3, 1, 13)
    34  }
    35  
    36  const testFunction = `
    37  def func(x):
    38      pass
    39  `
    40  
    41  func TestLexFunction(t *testing.T) {
    42  	l := newLexer(strings.NewReader(testFunction))
    43  	assertToken(t, l.Next(), Ident, "def", 2, 1, 2)
    44  	assertToken(t, l.Next(), Ident, "func", 2, 5, 6)
    45  	assertToken(t, l.Next(), '(', "(", 2, 9, 10)
    46  	assertToken(t, l.Next(), Ident, "x", 2, 10, 11)
    47  	assertToken(t, l.Next(), ')', ")", 2, 11, 12)
    48  	assertToken(t, l.Next(), ':', ":", 2, 12, 13)
    49  	assertToken(t, l.Next(), EOL, "", 2, 13, 14)
    50  	assertToken(t, l.Next(), Ident, "pass", 3, 5, 19)
    51  	assertToken(t, l.Next(), EOL, "", 4, 1, 23)
    52  	assertToken(t, l.Next(), Unindent, "", 4, 1, 24)
    53  	assertToken(t, l.Next(), EOF, "", 4, 1, 24)
    54  }
    55  
    56  func TestLexUnicode(t *testing.T) {
    57  	l := newLexer(strings.NewReader("懂了吗 你愁脸 有没有"))
    58  	assertToken(t, l.Next(), Ident, "懂了吗", 1, 1, 1)
    59  	assertToken(t, l.Next(), Ident, "你愁脸", 1, 11, 11)
    60  	assertToken(t, l.Next(), Ident, "有没有", 1, 21, 21)
    61  	assertToken(t, l.Next(), EOL, "", 1, 30, 30)
    62  	assertToken(t, l.Next(), EOF, "", 2, 1, 31)
    63  }
    64  
    65  func TestLexString(t *testing.T) {
    66  	l := newLexer(strings.NewReader(`x = "hello world"`))
    67  	assertToken(t, l.Next(), Ident, "x", 1, 1, 1)
    68  	assertToken(t, l.Next(), '=', "=", 1, 3, 3)
    69  	assertToken(t, l.Next(), String, "\"hello world\"", 1, 5, 5)
    70  	assertToken(t, l.Next(), EOL, "", 1, 18, 18)
    71  	assertToken(t, l.Next(), EOF, "", 2, 1, 19)
    72  }
    73  
    74  func TestLexStringEscape(t *testing.T) {
    75  	l := newLexer(strings.NewReader(`x = '\n\\'`))
    76  	assertToken(t, l.Next(), Ident, "x", 1, 1, 1)
    77  	assertToken(t, l.Next(), '=', "=", 1, 3, 3)
    78  	assertToken(t, l.Next(), String, "\"\n\\\"", 1, 5, 5)
    79  	assertToken(t, l.Next(), EOL, "", 1, 11, 11)
    80  	assertToken(t, l.Next(), EOF, "", 2, 1, 12)
    81  }
    82  
    83  func TestLexStringEscape2(t *testing.T) {
    84  	l := newLexer(strings.NewReader(`'echo -n "import \( \";'`))
    85  	assertToken(t, l.Next(), String, `"echo -n "import \( ";"`, 1, 1, 1)
    86  	assertToken(t, l.Next(), EOL, "", 1, 25, 25)
    87  	assertToken(t, l.Next(), EOF, "", 2, 1, 26)
    88  }
    89  
    90  func TestLexRawString(t *testing.T) {
    91  	l := newLexer(strings.NewReader(`x = r'\n\\'`))
    92  	assertToken(t, l.Next(), Ident, "x", 1, 1, 1)
    93  	assertToken(t, l.Next(), '=', "=", 1, 3, 3)
    94  	assertToken(t, l.Next(), String, `"\n\\"`, 1, 5, 5)
    95  	assertToken(t, l.Next(), EOL, "", 1, 12, 12)
    96  	assertToken(t, l.Next(), EOF, "", 2, 1, 13)
    97  }
    98  
    99  func TestLexFString(t *testing.T) {
   100  	l := newLexer(strings.NewReader(`x = f'{x}'`))
   101  	assertToken(t, l.Next(), Ident, "x", 1, 1, 1)
   102  	assertToken(t, l.Next(), '=', "=", 1, 3, 3)
   103  	assertToken(t, l.Next(), String, `f"{x}"`, 1, 5, 5)
   104  	assertToken(t, l.Next(), EOL, "", 1, 11, 11)
   105  	assertToken(t, l.Next(), EOF, "", 2, 1, 12)
   106  }
   107  
   108  const testMultilineString = `x = """
   109  hello\n
   110  world
   111  """`
   112  
   113  // expected output after lexing; note quotes are broken to a single one and \n does not become a newline.
   114  const expectedMultilineString = `"
   115  hello
   116  
   117  world
   118  "`
   119  
   120  func TestLexMultilineString(t *testing.T) {
   121  	l := newLexer(strings.NewReader(testMultilineString))
   122  	assertToken(t, l.Next(), Ident, "x", 1, 1, 1)
   123  	assertToken(t, l.Next(), '=', "=", 1, 3, 3)
   124  	assertToken(t, l.Next(), String, expectedMultilineString, 1, 5, 5)
   125  	assertToken(t, l.Next(), EOL, "", 4, 4, 26)
   126  	assertToken(t, l.Next(), EOF, "", 5, 1, 27)
   127  }
   128  
   129  func TestLexAttributeAccess(t *testing.T) {
   130  	l := newLexer(strings.NewReader(`x.call(y)`))
   131  	assertToken(t, l.Next(), Ident, "x", 1, 1, 1)
   132  	assertToken(t, l.Next(), '.', ".", 1, 2, 2)
   133  	assertToken(t, l.Next(), Ident, "call", 1, 3, 3)
   134  	assertToken(t, l.Next(), '(', "(", 1, 7, 7)
   135  	assertToken(t, l.Next(), Ident, "y", 1, 8, 8)
   136  	assertToken(t, l.Next(), ')', ")", 1, 9, 9)
   137  	assertToken(t, l.Next(), EOL, "", 1, 10, 10)
   138  	assertToken(t, l.Next(), EOF, "", 2, 1, 11)
   139  }
   140  
   141  func TestLexFunctionArgs(t *testing.T) {
   142  	l := newLexer(strings.NewReader(`def test(name='name', timeout=10, args=CONFIG.ARGS):`))
   143  	assertToken(t, l.Next(), Ident, "def", 1, 1, 1)
   144  	assertToken(t, l.Next(), Ident, "test", 1, 5, 5)
   145  	assertToken(t, l.Next(), '(', "(", 1, 9, 9)
   146  	assertToken(t, l.Next(), Ident, "name", 1, 10, 10)
   147  	assertToken(t, l.Next(), '=', "=", 1, 14, 14)
   148  	assertToken(t, l.Next(), String, "\"name\"", 1, 15, 15)
   149  	assertToken(t, l.Next(), ',', ",", 1, 21, 21)
   150  	assertToken(t, l.Next(), Ident, "timeout", 1, 23, 23)
   151  	assertToken(t, l.Next(), '=', "=", 1, 30, 30)
   152  	assertToken(t, l.Next(), Int, "10", 1, 31, 31)
   153  	assertToken(t, l.Next(), ',', ",", 1, 33, 33)
   154  	assertToken(t, l.Next(), Ident, "args", 1, 35, 35)
   155  	assertToken(t, l.Next(), '=', "=", 1, 39, 39)
   156  	assertToken(t, l.Next(), Ident, "CONFIG", 1, 40, 40)
   157  	assertToken(t, l.Next(), '.', ".", 1, 46, 46)
   158  	assertToken(t, l.Next(), Ident, "ARGS", 1, 47, 47)
   159  	assertToken(t, l.Next(), ')', ")", 1, 51, 51)
   160  	assertToken(t, l.Next(), ':', ":", 1, 52, 52)
   161  }
   162  
   163  const inputFunction = `
   164  python_library(
   165      name = 'lib',
   166      srcs = [
   167          'lib1.py',
   168          'lib2.py',
   169      ],
   170  )
   171  `
   172  
   173  func TestMoreComplexFunction(t *testing.T) {
   174  	l := newLexer(strings.NewReader(inputFunction))
   175  	assertToken(t, l.Next(), Ident, "python_library", 2, 1, 2)
   176  	assertToken(t, l.Next(), '(', "(", 2, 15, 16)
   177  	assertToken(t, l.Next(), Ident, "name", 3, 5, 22)
   178  	assertToken(t, l.Next(), '=', "=", 3, 10, 27)
   179  	assertToken(t, l.Next(), String, "\"lib\"", 3, 12, 29)
   180  	assertToken(t, l.Next(), ',', ",", 3, 17, 34)
   181  	assertToken(t, l.Next(), Ident, "srcs", 4, 5, 40)
   182  	assertToken(t, l.Next(), '=', "=", 4, 10, 45)
   183  	assertToken(t, l.Next(), '[', "[", 4, 12, 47)
   184  	assertToken(t, l.Next(), String, "\"lib1.py\"", 5, 9, 57)
   185  	assertToken(t, l.Next(), ',', ",", 5, 18, 66)
   186  	assertToken(t, l.Next(), String, "\"lib2.py\"", 6, 9, 76)
   187  	assertToken(t, l.Next(), ',', ",", 6, 18, 85)
   188  	assertToken(t, l.Next(), ']', "]", 7, 5, 91)
   189  	assertToken(t, l.Next(), ',', ",", 7, 6, 92)
   190  	assertToken(t, l.Next(), ')', ")", 8, 1, 94)
   191  }
   192  
   193  const multiUnindent = `
   194  for y in x:
   195      for z in y:
   196          for a in z:
   197              pass
   198  `
   199  
   200  func TestMultiUnindent(t *testing.T) {
   201  	l := newLexer(strings.NewReader(multiUnindent))
   202  	assertToken(t, l.Next(), Ident, "for", 2, 1, 2)
   203  	assertToken(t, l.Next(), Ident, "y", 2, 5, 6)
   204  	assertToken(t, l.Next(), Ident, "in", 2, 7, 8)
   205  	assertToken(t, l.Next(), Ident, "x", 2, 10, 11)
   206  	assertToken(t, l.Next(), ':', ":", 2, 11, 12)
   207  	assertToken(t, l.Next(), EOL, "", 2, 12, 13)
   208  	assertToken(t, l.Next(), Ident, "for", 3, 5, 18)
   209  	assertToken(t, l.Next(), Ident, "z", 3, 9, 22)
   210  	assertToken(t, l.Next(), Ident, "in", 3, 11, 24)
   211  	assertToken(t, l.Next(), Ident, "y", 3, 14, 27)
   212  	assertToken(t, l.Next(), ':', ":", 3, 15, 28)
   213  	assertToken(t, l.Next(), EOL, "", 3, 16, 29)
   214  	assertToken(t, l.Next(), Ident, "for", 4, 9, 38)
   215  	assertToken(t, l.Next(), Ident, "a", 4, 13, 42)
   216  	assertToken(t, l.Next(), Ident, "in", 4, 15, 44)
   217  	assertToken(t, l.Next(), Ident, "z", 4, 18, 47)
   218  	assertToken(t, l.Next(), ':', ":", 4, 19, 48)
   219  	assertToken(t, l.Next(), EOL, "", 4, 20, 49)
   220  	assertToken(t, l.Next(), Ident, "pass", 5, 13, 62)
   221  	assertToken(t, l.Next(), EOL, "", 6, 1, 66)
   222  	assertToken(t, l.Next(), Unindent, "", 6, 1, 67)
   223  	assertToken(t, l.Next(), Unindent, "", 6, 1, 67)
   224  	assertToken(t, l.Next(), Unindent, "", 6, 1, 67)
   225  }
   226  
   227  const multiLineFunctionArgs = `
   228  def test(name='name', timeout=10,
   229           args=CONFIG.ARGS):
   230      pass
   231  `
   232  
   233  func TestMultiLineFunctionArgs(t *testing.T) {
   234  	l := newLexer(strings.NewReader(multiLineFunctionArgs))
   235  	assertToken(t, l.Next(), Ident, "def", 2, 1, 2)
   236  	assertToken(t, l.Next(), Ident, "test", 2, 5, 6)
   237  	assertToken(t, l.Next(), '(', "(", 2, 9, 10)
   238  	assertToken(t, l.Next(), Ident, "name", 2, 10, 11)
   239  	assertToken(t, l.Next(), '=', "=", 2, 14, 15)
   240  	assertToken(t, l.Next(), String, "\"name\"", 2, 15, 16)
   241  	assertToken(t, l.Next(), ',', ",", 2, 21, 22)
   242  	assertToken(t, l.Next(), Ident, "timeout", 2, 23, 24)
   243  	assertToken(t, l.Next(), '=', "=", 2, 30, 31)
   244  	assertToken(t, l.Next(), Int, "10", 2, 31, 32)
   245  	assertToken(t, l.Next(), ',', ",", 2, 33, 34)
   246  	assertToken(t, l.Next(), Ident, "args", 3, 10, 45)
   247  	assertToken(t, l.Next(), '=', "=", 3, 14, 49)
   248  	assertToken(t, l.Next(), Ident, "CONFIG", 3, 15, 50)
   249  	assertToken(t, l.Next(), '.', ".", 3, 21, 56)
   250  	assertToken(t, l.Next(), Ident, "ARGS", 3, 22, 57)
   251  	assertToken(t, l.Next(), ')', ")", 3, 26, 61)
   252  	assertToken(t, l.Next(), ':', ":", 3, 27, 62)
   253  	assertToken(t, l.Next(), EOL, "", 3, 28, 63)
   254  	assertToken(t, l.Next(), Ident, "pass", 4, 5, 68)
   255  	assertToken(t, l.Next(), EOL, "", 5, 1, 72)
   256  	assertToken(t, l.Next(), Unindent, "", 5, 1, 73)
   257  }
   258  
   259  func TestComparisonOperator(t *testing.T) {
   260  	l := newLexer(strings.NewReader("x = y == z"))
   261  	assertToken(t, l.Next(), Ident, "x", 1, 1, 1)
   262  	assertToken(t, l.Next(), '=', "=", 1, 3, 3)
   263  	assertToken(t, l.Next(), Ident, "y", 1, 5, 5)
   264  	assertToken(t, l.Next(), LexOperator, "==", 1, 7, 7)
   265  }
   266  
   267  const blankLinesInFunction = `
   268  def x():
   269      """test"""
   270  
   271      return 42
   272  `
   273  
   274  func TestBlankLinesInFunction(t *testing.T) {
   275  	l := newLexer(strings.NewReader(blankLinesInFunction))
   276  	assertToken(t, l.Next(), Ident, "def", 2, 1, 2)
   277  	assertToken(t, l.Next(), Ident, "x", 2, 5, 6)
   278  	assertToken(t, l.Next(), '(', "(", 2, 6, 7)
   279  	assertToken(t, l.Next(), ')', ")", 2, 7, 8)
   280  	assertToken(t, l.Next(), ':', ":", 2, 8, 9)
   281  	assertToken(t, l.Next(), EOL, "", 2, 9, 10)
   282  	assertToken(t, l.Next(), String, "\"test\"", 3, 5, 15)
   283  	assertToken(t, l.Next(), EOL, "", 4, 1, 26)
   284  	assertToken(t, l.Next(), Ident, "return", 5, 5, 31)
   285  	assertToken(t, l.Next(), Int, "42", 5, 12, 38)
   286  	assertToken(t, l.Next(), EOL, "", 6, 1, 40)
   287  	assertToken(t, l.Next(), Unindent, "", 6, 1, 41)
   288  }
   289  
   290  const commentsAndEOLs = `
   291  pass
   292  
   293  # something
   294  
   295  `
   296  
   297  func TestCommentsAndEOLs(t *testing.T) {
   298  	l := newLexer(strings.NewReader(commentsAndEOLs))
   299  	assertToken(t, l.Next(), Ident, "pass", 2, 1, 2)
   300  	assertToken(t, l.Next(), EOL, "", 3, 1, 7)
   301  	assertToken(t, l.Next(), EOF, "", 6, 1, 21)
   302  }
   303  
   304  // This is a much-simplified version of the true motivating case.
   305  const unevenIndent = `
   306  def x():
   307      if True:
   308              pass
   309      return
   310  `
   311  
   312  func TestUnevenUnindent(t *testing.T) {
   313  	l := newLexer(strings.NewReader(unevenIndent))
   314  	assertToken(t, l.Next(), Ident, "def", 2, 1, 2)
   315  	assertToken(t, l.Next(), Ident, "x", 2, 5, 6)
   316  	assertToken(t, l.Next(), '(', "(", 2, 6, 7)
   317  	assertToken(t, l.Next(), ')', ")", 2, 7, 8)
   318  	assertToken(t, l.Next(), ':', ":", 2, 8, 9)
   319  	assertToken(t, l.Next(), EOL, "", 2, 9, 10)
   320  	assertToken(t, l.Next(), Ident, "if", 3, 5, 15)
   321  	assertToken(t, l.Next(), Ident, "True", 3, 8, 18)
   322  	assertToken(t, l.Next(), ':', ":", 3, 12, 22)
   323  	assertToken(t, l.Next(), EOL, "", 3, 13, 23)
   324  	assertToken(t, l.Next(), Ident, "pass", 4, 13, 36)
   325  	assertToken(t, l.Next(), EOL, "", 5, 5, 40)
   326  	assertToken(t, l.Next(), Unindent, "", 5, 5, 45)
   327  	assertToken(t, l.Next(), Ident, "return", 5, 5, 45)
   328  	assertToken(t, l.Next(), EOL, "", 6, 1, 51)
   329  	assertToken(t, l.Next(), Unindent, "", 6, 1, 52)
   330  	assertToken(t, l.Next(), EOF, "", 6, 1, 52)
   331  }
   332  
   333  const implicitStringConcatenation = `
   334  str('testing that we can carry these '
   335      'over multiple lines')
   336  `
   337  
   338  func TestImplicitStringConcatenation(t *testing.T) {
   339  	l := newLexer(strings.NewReader(implicitStringConcatenation))
   340  	assertToken(t, l.Next(), Ident, "str", 2, 1, 2)
   341  	assertToken(t, l.Next(), '(', "(", 2, 4, 5)
   342  	assertToken(t, l.Next(), String, `"testing that we can carry these over multiple lines"`, 2, 5, 6)
   343  }
   344  
   345  func TestImplicitStringConcatenationOnlyHappensInsideBraces(t *testing.T) {
   346  	l := newLexer(strings.NewReader("'hello' 'world'"))
   347  	assertToken(t, l.Next(), String, `"hello"`, 1, 1, 1)
   348  	assertToken(t, l.Next(), String, `"world"`, 1, 9, 9)
   349  }