github.com/influx6/npkg@v0.8.8/nlexing/nlexing.go (about)

     1  package nlexing
     2  
     3  import (
     4  	"unicode/utf8"
     5  )
     6  
     7  const eof = rune(0)
     8  
     9  type TokenType int
    10  
    11  type ResultFunc func(b string, t TokenType) error
    12  
    13  type TokenFunc func(l *Lexer, res ResultFunc) (TokenFunc, error)
    14  
    15  type Tokenizer struct {
    16  	l        *Lexer
    17  	start    TokenFunc
    18  	resultFn ResultFunc
    19  }
    20  
    21  func NewTokenizer(l *Lexer, starter TokenFunc, results ResultFunc) *Tokenizer {
    22  	return &Tokenizer{
    23  		l:        l,
    24  		start:    starter,
    25  		resultFn: results,
    26  	}
    27  }
    28  
    29  func (t *Tokenizer) Run() error {
    30  	var stateFn = t.start
    31  	var err error
    32  	for {
    33  		if stateFn == nil {
    34  			break
    35  		}
    36  		stateFn, err = stateFn(t.l, t.resultFn)
    37  	}
    38  	return err
    39  }
    40  
    41  type Lexer struct {
    42  	pos   int
    43  	width int
    44  	start int
    45  	input string
    46  }
    47  
    48  func NewLexer(input string) *Lexer {
    49  	return &Lexer{
    50  		pos:   0,
    51  		width: 0,
    52  		start: 0,
    53  		input: input,
    54  	}
    55  }
    56  
    57  func (b *Lexer) isAtEnd() bool {
    58  	return b.pos >= len(b.input)
    59  }
    60  
    61  func (b *Lexer) backup() {
    62  	b.pos -= b.width
    63  }
    64  
    65  func (b *Lexer) next() rune {
    66  	if b.pos >= len(b.input) {
    67  		b.width = 0
    68  		return eof
    69  	}
    70  	var rn, width = utf8.DecodeRuneInString(b.input[b.pos:])
    71  	b.width = width
    72  	b.pos += width
    73  	return rn
    74  }
    75  
    76  func (b *Lexer) skipNext() rune {
    77  	var nr = b.next()
    78  	b.ignore()
    79  	return nr
    80  }
    81  
    82  func (b *Lexer) peek() rune {
    83  	var nr = b.next()
    84  	b.backup()
    85  	return nr
    86  }
    87  
    88  func (b *Lexer) skipRune() {
    89  	b.start += b.width
    90  }
    91  
    92  func (b *Lexer) ignore() {
    93  	b.start = b.pos
    94  }
    95  
    96  func (b *Lexer) peekSlice() string {
    97  	if b.pos >= len(b.input) {
    98  		return b.input[b.start:]
    99  	}
   100  	var sl = b.input[b.start:b.pos]
   101  	return sl
   102  }
   103  
   104  func (b *Lexer) slice() string {
   105  	if b.pos >= len(b.input) {
   106  		return b.input[b.start:]
   107  	}
   108  	var sl = b.input[b.start:b.pos]
   109  	b.start = b.pos
   110  	return sl
   111  }