modernc.org/ql@v1.4.7/scanner_test.go (about) 1 // Copyright (c) 2014 ql Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style 3 // license that can be found in the LICENSE file. 4 5 package ql // import "modernc.org/ql" 6 7 import ( 8 "fmt" 9 "testing" 10 "unicode" 11 ) 12 13 func tok2name(i int) string { 14 if i == unicode.ReplacementChar { 15 return "<?>" 16 } 17 18 if i < 128 { 19 return fmt.Sprintf("tok-'%c'", i) 20 } 21 22 return fmt.Sprintf("tok-%d", i) 23 } 24 25 func TestScanner0(t *testing.T) { 26 table := []struct { 27 src string 28 tok, line, col, nline, ncol int 29 val string 30 }{ 31 {"a", identifier, 1, 1, 1, 2, "a"}, 32 {" a", identifier, 1, 2, 1, 3, "a"}, 33 {"a ", identifier, 1, 1, 1, 2, "a"}, 34 {" a ", identifier, 1, 2, 1, 3, "a"}, 35 {"\na", identifier, 2, 1, 2, 2, "a"}, 36 37 {"a\n", identifier, 1, 1, 1, 2, "a"}, 38 {"\na\n", identifier, 2, 1, 2, 2, "a"}, 39 {"\n a", identifier, 2, 2, 2, 3, "a"}, 40 {"a \n", identifier, 1, 1, 1, 2, "a"}, 41 {"\n a \n", identifier, 2, 2, 2, 3, "a"}, 42 43 {"ab", identifier, 1, 1, 1, 3, "ab"}, 44 {" ab", identifier, 1, 2, 1, 4, "ab"}, 45 {"ab ", identifier, 1, 1, 1, 3, "ab"}, 46 {" ab ", identifier, 1, 2, 1, 4, "ab"}, 47 {"\nab", identifier, 2, 1, 2, 3, "ab"}, 48 49 {"ab\n", identifier, 1, 1, 1, 3, "ab"}, 50 {"\nab\n", identifier, 2, 1, 2, 3, "ab"}, 51 {"\n ab", identifier, 2, 2, 2, 4, "ab"}, 52 {"ab \n", identifier, 1, 1, 1, 3, "ab"}, 53 {"\n ab \n", identifier, 2, 2, 2, 4, "ab"}, 54 55 {"c", identifier, 1, 1, 1, 2, "c"}, 56 {"cR", identifier, 1, 1, 1, 3, "cR"}, 57 {"cRe", identifier, 1, 1, 1, 4, "cRe"}, 58 {"cReA", identifier, 1, 1, 1, 5, "cReA"}, 59 {"cReAt", identifier, 1, 1, 1, 6, "cReAt"}, 60 61 {"cReATe", create, 1, 1, 1, 7, "cReATe"}, 62 {"cReATeD", identifier, 1, 1, 1, 8, "cReATeD"}, 63 {"2", intLit, 1, 1, 1, 2, "2"}, 64 {"2.", floatLit, 1, 1, 1, 3, "2."}, 65 {"2.3", floatLit, 1, 1, 1, 4, "2.3"}, 66 67 {"ááá ", identifier, 1, 1, 1, 7, "ááá"}, 68 69 {`«id with spaces»`, identifier, 1, 1, 1, 19, "«id with spaces»"}, 70 {`«TRANSACTION»`, identifier, 1, 1, 1, 16, "«TRANSACTION»"}, 71 {"«-+ \t!\"#$%&'()*,./:;<=>?@[\\]^{|}~=>`»", identifier, 72 1, 1, 1, 40, "«-+ \t!\"#$%&'()*,./:;<=>?@[\\]^{|}~=>`»"}, 73 {`«€/µg»`, identifier, 1, 1, 1, 12, "«€/µg»"}, 74 } 75 76 lval := &yySymType{} 77 for i, test := range table { 78 l, err := newLexer(test.src) 79 if err != nil { 80 t.Fatal(err) 81 } 82 83 tok := l.Lex(lval) 84 nline, ncol := l.npos() 85 val := string(l.TokenBytes(nil)) 86 if tok != test.tok || l.line != test.line || l.col != test.col || 87 nline != test.nline || ncol != test.ncol || 88 val != test.val { 89 t.Fatalf( 90 "%d g: %s %d:%d-%d:%d %q, e: %s %d:%d-%d:%d %q", 91 i, tok2name(tok), l.line, l.col, nline, ncol, val, 92 tok2name(test.tok), test.line, test.col, test.nline, test.ncol, test.val, 93 ) 94 } 95 } 96 }