modernc.org/ql@v1.4.7/scanner_test.go (about)

     1  // Copyright (c) 2014 ql Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  package ql // import "modernc.org/ql"
     6  
     7  import (
     8  	"fmt"
     9  	"testing"
    10  	"unicode"
    11  )
    12  
    13  func tok2name(i int) string {
    14  	if i == unicode.ReplacementChar {
    15  		return "<?>"
    16  	}
    17  
    18  	if i < 128 {
    19  		return fmt.Sprintf("tok-'%c'", i)
    20  	}
    21  
    22  	return fmt.Sprintf("tok-%d", i)
    23  }
    24  
    25  func TestScanner0(t *testing.T) {
    26  	table := []struct {
    27  		src                         string
    28  		tok, line, col, nline, ncol int
    29  		val                         string
    30  	}{
    31  		{"a", identifier, 1, 1, 1, 2, "a"},
    32  		{" a", identifier, 1, 2, 1, 3, "a"},
    33  		{"a ", identifier, 1, 1, 1, 2, "a"},
    34  		{" a ", identifier, 1, 2, 1, 3, "a"},
    35  		{"\na", identifier, 2, 1, 2, 2, "a"},
    36  
    37  		{"a\n", identifier, 1, 1, 1, 2, "a"},
    38  		{"\na\n", identifier, 2, 1, 2, 2, "a"},
    39  		{"\n a", identifier, 2, 2, 2, 3, "a"},
    40  		{"a \n", identifier, 1, 1, 1, 2, "a"},
    41  		{"\n a \n", identifier, 2, 2, 2, 3, "a"},
    42  
    43  		{"ab", identifier, 1, 1, 1, 3, "ab"},
    44  		{" ab", identifier, 1, 2, 1, 4, "ab"},
    45  		{"ab ", identifier, 1, 1, 1, 3, "ab"},
    46  		{" ab ", identifier, 1, 2, 1, 4, "ab"},
    47  		{"\nab", identifier, 2, 1, 2, 3, "ab"},
    48  
    49  		{"ab\n", identifier, 1, 1, 1, 3, "ab"},
    50  		{"\nab\n", identifier, 2, 1, 2, 3, "ab"},
    51  		{"\n ab", identifier, 2, 2, 2, 4, "ab"},
    52  		{"ab \n", identifier, 1, 1, 1, 3, "ab"},
    53  		{"\n ab \n", identifier, 2, 2, 2, 4, "ab"},
    54  
    55  		{"c", identifier, 1, 1, 1, 2, "c"},
    56  		{"cR", identifier, 1, 1, 1, 3, "cR"},
    57  		{"cRe", identifier, 1, 1, 1, 4, "cRe"},
    58  		{"cReA", identifier, 1, 1, 1, 5, "cReA"},
    59  		{"cReAt", identifier, 1, 1, 1, 6, "cReAt"},
    60  
    61  		{"cReATe", create, 1, 1, 1, 7, "cReATe"},
    62  		{"cReATeD", identifier, 1, 1, 1, 8, "cReATeD"},
    63  		{"2", intLit, 1, 1, 1, 2, "2"},
    64  		{"2.", floatLit, 1, 1, 1, 3, "2."},
    65  		{"2.3", floatLit, 1, 1, 1, 4, "2.3"},
    66  
    67  		{"ááá ", identifier, 1, 1, 1, 7, "ááá"},
    68  
    69  		{`«id with spaces»`, identifier, 1, 1, 1, 19, "«id with spaces»"},
    70  		{`«TRANSACTION»`, identifier, 1, 1, 1, 16, "«TRANSACTION»"},
    71  		{"«-+ \t!\"#$%&'()*,./:;<=>?@[\\]^{|}~=>`»", identifier,
    72  			1, 1, 1, 40, "«-+ \t!\"#$%&'()*,./:;<=>?@[\\]^{|}~=>`»"},
    73  		{`«€/µg»`, identifier, 1, 1, 1, 12, "«€/µg»"},
    74  	}
    75  
    76  	lval := &yySymType{}
    77  	for i, test := range table {
    78  		l, err := newLexer(test.src)
    79  		if err != nil {
    80  			t.Fatal(err)
    81  		}
    82  
    83  		tok := l.Lex(lval)
    84  		nline, ncol := l.npos()
    85  		val := string(l.TokenBytes(nil))
    86  		if tok != test.tok || l.line != test.line || l.col != test.col ||
    87  			nline != test.nline || ncol != test.ncol ||
    88  			val != test.val {
    89  			t.Fatalf(
    90  				"%d g: %s %d:%d-%d:%d %q, e: %s %d:%d-%d:%d %q",
    91  				i, tok2name(tok), l.line, l.col, nline, ncol, val,
    92  				tok2name(test.tok), test.line, test.col, test.nline, test.ncol, test.val,
    93  			)
    94  		}
    95  	}
    96  }