modernc.org/ql@v1.4.7/lexer.go (about) 1 // Copyright 2017 The ql Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style 3 // license that can be found in the LICENSE file. 4 5 package ql // import "modernc.org/ql" 6 7 import ( 8 "fmt" 9 "go/scanner" 10 "go/token" 11 "math" 12 "strconv" 13 "strings" 14 "unicode" 15 16 "modernc.org/golex/lex" 17 ) 18 19 const ( 20 ccEOF = iota + 0x80 21 ccLetter 22 ccDigit 23 ccOther 24 ccOGuill 25 ccCGuill 26 ) 27 28 func runeClass(r rune) int { 29 switch { 30 case r == lex.RuneEOF: 31 return ccEOF 32 case r < 0x80: 33 return int(r) 34 case unicode.IsLetter(r): 35 return ccLetter 36 case unicode.IsDigit(r): 37 return ccDigit 38 case r == '«': 39 return ccOGuill 40 case r == '»': 41 return ccCGuill 42 default: 43 return ccOther 44 } 45 } 46 47 type lexer struct { 48 *lex.Lexer 49 agg []bool 50 col int 51 errs scanner.ErrorList 52 expr expression 53 file *token.File 54 inj int 55 line int 56 list []stmt 57 params int 58 root bool 59 sc int 60 } 61 62 func newLexer(src string) (*lexer, error) { 63 fset := token.NewFileSet() 64 file := fset.AddFile("", -1, len(src)) 65 l := &lexer{ 66 file: file, 67 } 68 l0, err := lex.New( 69 file, 70 strings.NewReader(src), 71 lex.ErrorFunc(func(pos token.Pos, msg string) { 72 l.errPos(pos, msg) 73 }), 74 lex.RuneClass(runeClass), 75 lex.BOMMode(lex.BOMIgnoreFirst), 76 ) 77 if err != nil { 78 return nil, err 79 } 80 81 l.Lexer = l0 82 return l, nil 83 } 84 85 func (l *lexer) errPos(pos token.Pos, format string, arg ...interface{}) { 86 l.errs.Add(l.file.Position(pos), fmt.Sprintf(format, arg...)) 87 } 88 89 func (l *lexer) err(s string, arg ...interface{}) { 90 l.errPos(l.Last.Pos(), s, arg...) 91 } 92 93 // Implements yyLexer. 94 func (l *lexer) Error(s string) { 95 l.err(s) 96 } 97 98 func (l *lexer) int(lval *yySymType, im bool) int { 99 val := l.TokenBytes(nil) 100 if im { 101 val = val[:len(val)-1] 102 } 103 n, err := strconv.ParseUint(string(val), 0, 64) 104 if err != nil { 105 l.err("integer literal: %v", err) 106 return int(unicode.ReplacementChar) 107 } 108 109 if im { 110 lval.item = idealComplex(complex(0, float64(n))) 111 return imaginaryLit 112 } 113 114 switch { 115 case n < math.MaxInt64: 116 lval.item = idealInt(n) 117 default: 118 lval.item = idealUint(n) 119 } 120 return intLit 121 } 122 123 func (l *lexer) float(lval *yySymType, im bool) int { 124 val := l.TokenBytes(nil) 125 if im { 126 val = val[:len(val)-1] 127 } 128 n, err := strconv.ParseFloat(string(val), 64) 129 if err != nil { 130 l.err("float literal: %v", err) 131 return int(unicode.ReplacementChar) 132 } 133 134 if im { 135 lval.item = idealComplex(complex(0, n)) 136 return imaginaryLit 137 } 138 139 lval.item = idealFloat(n) 140 return floatLit 141 } 142 143 func (l *lexer) str(lval *yySymType, pref string) int { 144 val := l.TokenBytes(nil) 145 l.sc = 0 146 s := pref + string(val) 147 s, err := strconv.Unquote(s) 148 if err != nil { 149 l.err("string literal: %v", err) 150 return int(unicode.ReplacementChar) 151 } 152 153 lval.item = s 154 return stringLit 155 } 156 157 func (l *lexer) delimitedIdentifier(lval *yySymType) int { 158 val := l.TokenBytes(nil) 159 l.sc = 0 160 if len(val) < 5 { 161 l.err("quotedIdentifier too short: %v", val) 162 return int(unicode.ReplacementChar) 163 } 164 s := string(val[2:(len(val) - 2)]) 165 lval.item = s 166 return identifier 167 } 168 169 func (l *lexer) npos() (line, col int) { 170 pos := l.file.Position(l.Last.Pos()) 171 return pos.Line, pos.Column 172 }