modernc.org/ql@v1.4.7/lexer.go (about)

     1  // Copyright 2017 The ql Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  package ql // import "modernc.org/ql"
     6  
     7  import (
     8  	"fmt"
     9  	"go/scanner"
    10  	"go/token"
    11  	"math"
    12  	"strconv"
    13  	"strings"
    14  	"unicode"
    15  
    16  	"modernc.org/golex/lex"
    17  )
    18  
    19  const (
    20  	ccEOF = iota + 0x80
    21  	ccLetter
    22  	ccDigit
    23  	ccOther
    24  	ccOGuill
    25  	ccCGuill
    26  )
    27  
    28  func runeClass(r rune) int {
    29  	switch {
    30  	case r == lex.RuneEOF:
    31  		return ccEOF
    32  	case r < 0x80:
    33  		return int(r)
    34  	case unicode.IsLetter(r):
    35  		return ccLetter
    36  	case unicode.IsDigit(r):
    37  		return ccDigit
    38  	case r == '«':
    39  		return ccOGuill
    40  	case r == '»':
    41  		return ccCGuill
    42  	default:
    43  		return ccOther
    44  	}
    45  }
    46  
    47  type lexer struct {
    48  	*lex.Lexer
    49  	agg    []bool
    50  	col    int
    51  	errs   scanner.ErrorList
    52  	expr   expression
    53  	file   *token.File
    54  	inj    int
    55  	line   int
    56  	list   []stmt
    57  	params int
    58  	root   bool
    59  	sc     int
    60  }
    61  
    62  func newLexer(src string) (*lexer, error) {
    63  	fset := token.NewFileSet()
    64  	file := fset.AddFile("", -1, len(src))
    65  	l := &lexer{
    66  		file: file,
    67  	}
    68  	l0, err := lex.New(
    69  		file,
    70  		strings.NewReader(src),
    71  		lex.ErrorFunc(func(pos token.Pos, msg string) {
    72  			l.errPos(pos, msg)
    73  		}),
    74  		lex.RuneClass(runeClass),
    75  		lex.BOMMode(lex.BOMIgnoreFirst),
    76  	)
    77  	if err != nil {
    78  		return nil, err
    79  	}
    80  
    81  	l.Lexer = l0
    82  	return l, nil
    83  }
    84  
    85  func (l *lexer) errPos(pos token.Pos, format string, arg ...interface{}) {
    86  	l.errs.Add(l.file.Position(pos), fmt.Sprintf(format, arg...))
    87  }
    88  
    89  func (l *lexer) err(s string, arg ...interface{}) {
    90  	l.errPos(l.Last.Pos(), s, arg...)
    91  }
    92  
    93  // Implements yyLexer.
    94  func (l *lexer) Error(s string) {
    95  	l.err(s)
    96  }
    97  
    98  func (l *lexer) int(lval *yySymType, im bool) int {
    99  	val := l.TokenBytes(nil)
   100  	if im {
   101  		val = val[:len(val)-1]
   102  	}
   103  	n, err := strconv.ParseUint(string(val), 0, 64)
   104  	if err != nil {
   105  		l.err("integer literal: %v", err)
   106  		return int(unicode.ReplacementChar)
   107  	}
   108  
   109  	if im {
   110  		lval.item = idealComplex(complex(0, float64(n)))
   111  		return imaginaryLit
   112  	}
   113  
   114  	switch {
   115  	case n < math.MaxInt64:
   116  		lval.item = idealInt(n)
   117  	default:
   118  		lval.item = idealUint(n)
   119  	}
   120  	return intLit
   121  }
   122  
   123  func (l *lexer) float(lval *yySymType, im bool) int {
   124  	val := l.TokenBytes(nil)
   125  	if im {
   126  		val = val[:len(val)-1]
   127  	}
   128  	n, err := strconv.ParseFloat(string(val), 64)
   129  	if err != nil {
   130  		l.err("float literal: %v", err)
   131  		return int(unicode.ReplacementChar)
   132  	}
   133  
   134  	if im {
   135  		lval.item = idealComplex(complex(0, n))
   136  		return imaginaryLit
   137  	}
   138  
   139  	lval.item = idealFloat(n)
   140  	return floatLit
   141  }
   142  
   143  func (l *lexer) str(lval *yySymType, pref string) int {
   144  	val := l.TokenBytes(nil)
   145  	l.sc = 0
   146  	s := pref + string(val)
   147  	s, err := strconv.Unquote(s)
   148  	if err != nil {
   149  		l.err("string literal: %v", err)
   150  		return int(unicode.ReplacementChar)
   151  	}
   152  
   153  	lval.item = s
   154  	return stringLit
   155  }
   156  
   157  func (l *lexer) delimitedIdentifier(lval *yySymType) int {
   158  	val := l.TokenBytes(nil)
   159  	l.sc = 0
   160  	if len(val) < 5 {
   161  		l.err("quotedIdentifier too short: %v", val)
   162  		return int(unicode.ReplacementChar)
   163  	}
   164  	s := string(val[2:(len(val) - 2)])
   165  	lval.item = s
   166  	return identifier
   167  }
   168  
   169  func (l *lexer) npos() (line, col int) {
   170  	pos := l.file.Position(l.Last.Pos())
   171  	return pos.Line, pos.Column
   172  }