github.com/quantosnetwork/Quantos@v0.0.0-20220306172517-e20b28c5a29a/quantix/token/tokens.go (about)

     1  package token
     2  
     3  import (
     4  	"fmt"
     5  )
     6  
     7  type TokenType uint32
     8  
     9  const (
    10  	STRING TokenType = iota
    11  	NUMBER
    12  	IDENTIFIER
    13  	LEFT_PAREN
    14  	RIGHT_PAREN
    15  	LEFT_BRACE
    16  	RIGHT_BRACE
    17  	COMMA
    18  	DOT
    19  	MINUS
    20  	PLUS
    21  	SEMICOLON
    22  	SLASH
    23  	STAR
    24  	BANG
    25  	BANG_EQUAL
    26  	LESS
    27  	LESS_EQUAL
    28  	GREATER
    29  	GREATER_EQUAL
    30  	AND
    31  	CONTRACT
    32  	ABSTRACT
    33  	INTERFACE
    34  	LIB
    35  	FUNCTION
    36  	TRUE
    37  	FALSE
    38  	ELSE
    39  	FOR
    40  	IF
    41  	NIL
    42  	OR
    43  	PRINT
    44  	RETURN
    45  	SUPER
    46  	THIS
    47  	VAR
    48  	LET
    49  	CONST
    50  	WHILE
    51  	ADDRESS
    52  	PROXY
    53  	UINT256
    54  	UINT512
    55  	BRIDGE
    56  	PRAGMA
    57  	VERSION
    58  	SYNTAX
    59  )
    60  
    61  type Token struct {
    62  	typ        Type
    63  	lexeme     string
    64  	lext, rext int
    65  	literal    interface{}
    66  	line       int
    67  	input      []rune
    68  }
    69  
    70  func NewToken(typ TokenType, lext, rext int, input []rune) *Token {
    71  
    72  	return &Token{
    73  		typ, "", lext, rext, nil, 0, input,
    74  	}
    75  
    76  }
    77  
    78  // GetLineColumn returns the line and column of the left extent of t
    79  func (t *Token) GetLineColumn() (line, col int) {
    80  	line, col = 1, 1
    81  	for j := 0; j < t.lext; j++ {
    82  		switch t.input[j] {
    83  		case '\n':
    84  			line++
    85  			col = 1
    86  		case '\t':
    87  			col += 4
    88  		default:
    89  			col++
    90  		}
    91  	}
    92  	return
    93  }
    94  
    95  // GetInput returns the input from which t was parsed.
    96  func (t *Token) GetInput() []rune {
    97  	return t.input
    98  }
    99  
   100  // Lext returns the left extent of t
   101  func (t *Token) Lext() int {
   102  	return t.lext
   103  }
   104  
   105  // Literal returns the literal runes of t scanned by the lexer
   106  func (t *Token) Literal() []rune {
   107  	return t.input[t.lext:t.rext]
   108  }
   109  
   110  // LiteralString returns string(t.Literal())
   111  func (t *Token) LiteralString() string {
   112  	return string(t.Literal())
   113  }
   114  
   115  // LiteralStripEscape returns the literal runes of t scanned by the lexer
   116  func (t *Token) LiteralStripEscape() []rune {
   117  	lit := t.Literal()
   118  	strip := make([]rune, 0, len(lit))
   119  	for i := 0; i < len(lit); i++ {
   120  		if lit[i] == '\\' {
   121  			i++
   122  			switch lit[i] {
   123  			case 't':
   124  				strip = append(strip, '\t')
   125  			case 'r':
   126  				strip = append(strip, '\r')
   127  			case 'n':
   128  				strip = append(strip, '\r')
   129  			default:
   130  				strip = append(strip, lit[i])
   131  			}
   132  		} else {
   133  			strip = append(strip, lit[i])
   134  		}
   135  	}
   136  	return strip
   137  }
   138  
   139  // LiteralStringStripEscape returns string(t.LiteralStripEscape())
   140  func (t *Token) LiteralStringStripEscape() string {
   141  	return string(t.LiteralStripEscape())
   142  }
   143  
   144  // Rext returns the right extent of t in the input
   145  func (t *Token) Rext() int {
   146  	return t.rext
   147  }
   148  
   149  func (t *Token) String() string {
   150  	return fmt.Sprintf("%s (%d,%d) %s",
   151  		t.TypeID(), t.lext, t.rext, t.LiteralString())
   152  }
   153  
   154  // Suppress returns true iff t is suppressed by the lexer
   155  func (t *Token) Suppress() bool {
   156  	return Suppress[t.typ]
   157  }
   158  
   159  // Type returns the token Type of t
   160  func (t *Token) Type() Type {
   161  	return t.typ
   162  }
   163  
   164  // TypeID returns the token Type ID of t.
   165  // This may be different from the literal of token t.
   166  func (t *Token) TypeID() string {
   167  	return t.Type().ID()
   168  }