github.com/searKing/golang/go@v1.2.117/go/token/tokenizer.go (about)

     1  // Copyright 2020 The searKing Author. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  package token
     6  
     7  import (
     8  	"fmt"
     9  )
    10  
    11  var (
    12  	Literals = map[string]struct{}{"<": {}, ">": {}}
    13  )
    14  
    15  type TokenizerFunc interface {
    16  	IsLiteral() bool
    17  	IsOperator() bool
    18  	IsKeyword() bool
    19  }
    20  
    21  // type <value>, type <value>
    22  func Tokenizer(inputs []rune, f func(inputs []rune, current int) (token Token, next int), strict bool) []Token {
    23  	// type <key, value>
    24  	current := 0
    25  	var tokens []Token
    26  	for current < len(inputs) {
    27  		var token Token
    28  		token, current = f(inputs, current)
    29  		tokens = append(tokens, token)
    30  
    31  		if token.Typ == TypeILLEGAL && strict {
    32  			panic(fmt.Sprintf("I dont know what this token is: %s", token.Value))
    33  		}
    34  
    35  		if token.Typ == TypeEOF {
    36  			return tokens
    37  		}
    38  	}
    39  	return tokens
    40  }