github.com/neugram/ng@v0.0.0-20180309130942-d472ff93d872/syntax/token/token.go (about)

     1  // Copyright 2015 The Neugram Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  // Package token defines data structures representing Neugram tokens.
     6  package token
     7  
     8  import "fmt"
     9  
    10  // Token is a neugram lexical token.
    11  type Token int
    12  
    13  const (
    14  	Unknown Token = iota
    15  	Comment
    16  
    17  	// Constants
    18  
    19  	Ident     // E.g. funcName
    20  	Int       // E.g. 1001 TODO: rename to Integer?
    21  	Float     // E.g. 10.01
    22  	Imaginary // E.g. 10.01i
    23  	String    // E.g. "a string"
    24  	Rune      // E.g. '\u1f4a9'
    25  
    26  	// Expression Operators
    27  
    28  	Add          // +
    29  	Sub          // -
    30  	Mul          // *
    31  	Div          // /
    32  	Rem          // %
    33  	Pow          // ^
    34  	Ref          // &
    35  	RefPow       // &^
    36  	LogicalAnd   // &&
    37  	LogicalOr    // ||
    38  	Equal        // ==
    39  	Less         // <
    40  	Greater      // >
    41  	Assign       // =
    42  	Not          // !
    43  	NotEqual     // !=
    44  	LessEqual    // <=
    45  	GreaterEqual // >=
    46  	Shell        // $$
    47  	ShellWord    // [^\s|&;<>()]+
    48  	ShellPipe    // |
    49  	ShellNewline // \n
    50  	GreaterAnd   // >&
    51  	AndGreater   // &>
    52  	TwoGreater   // >>
    53  	TwoLess      // <<
    54  	ChanOp       // <-
    55  	Ellipsis     // ...
    56  
    57  	// Statement Operators
    58  
    59  	Inc       // ++
    60  	Dec       // --
    61  	AddAssign // +=
    62  	SubAssign // -=
    63  	MulAssign // *=
    64  	DivAssign // /=
    65  	RemAssign // %=
    66  	PowAssign // ^=
    67  	Define    // :=
    68  
    69  	LeftParen       // (
    70  	LeftBracket     // [
    71  	LeftBrace       // {
    72  	LeftBraceTable  // {|
    73  	RightParen      // )
    74  	RightBracket    // ]
    75  	RightBrace      // }
    76  	RightBraceTable // |}
    77  	Comma           // ,
    78  	Period          // .
    79  	Semicolon       // ;
    80  	Colon           // :
    81  	Pipe            // |
    82  
    83  	// Keywords
    84  
    85  	Package
    86  	Import
    87  
    88  	Func
    89  	Return
    90  	Defer
    91  
    92  	Select
    93  	Switch
    94  	Case
    95  	Default
    96  	Fallthrough
    97  
    98  	Const
    99  	Var
   100  
   101  	If
   102  	Else
   103  
   104  	For
   105  	Range
   106  	Continue
   107  	Break
   108  	Goto
   109  
   110  	Go
   111  
   112  	Chan
   113  	Map
   114  	Struct
   115  	Methodik
   116  	Interface
   117  	Type
   118  )
   119  
   120  var tokens = map[string]Token{
   121  	"unknown":      Unknown,
   122  	"comment":      Comment,
   123  	"ident":        Ident,
   124  	"integer":      Int,
   125  	"float":        Float,
   126  	"Imaginary":    Imaginary,
   127  	"string":       String,
   128  	"rune":         Rune,
   129  	"+":            Add,
   130  	"-":            Sub,
   131  	"*":            Mul,
   132  	"/":            Div,
   133  	"%":            Rem,
   134  	"^":            Pow,
   135  	"&":            Ref,
   136  	"&^":           RefPow,
   137  	"&&":           LogicalAnd,
   138  	"||":           LogicalOr,
   139  	"==":           Equal,
   140  	"<":            Less,
   141  	">":            Greater,
   142  	"=":            Assign,
   143  	"!":            Not,
   144  	"!=":           NotEqual,
   145  	"<=":           LessEqual,
   146  	">=":           GreaterEqual,
   147  	"$$":           Shell,
   148  	"shellword":    ShellWord,
   149  	"shellpipe":    ShellPipe, // TODO: use Pipe
   150  	"shellnewline": ShellNewline,
   151  	">&":           GreaterAnd,
   152  	"&>":           AndGreater,
   153  	">>":           TwoGreater,
   154  	"<<":           TwoLess,
   155  	"<-":           ChanOp,
   156  	"...":          Ellipsis,
   157  	"++":           Inc,
   158  	"--":           Dec,
   159  	"+=":           AddAssign,
   160  	"-=":           SubAssign,
   161  	"*=":           MulAssign,
   162  	"/=":           DivAssign,
   163  	"RemAssign":    RemAssign,
   164  	"PowAssign":    PowAssign,
   165  	":=":           Define,
   166  	"(":            LeftParen,
   167  	"[":            LeftBracket,
   168  	"{":            LeftBrace,
   169  	"{|":           LeftBraceTable,
   170  	")":            RightParen,
   171  	"]":            RightBracket,
   172  	"|}":           RightBraceTable,
   173  	",":            Comma,
   174  	".":            Period,
   175  	";":            Semicolon,
   176  	":":            Colon,
   177  	"|":            Pipe,
   178  }
   179  
   180  var Keywords = map[string]Token{
   181  	"package":     Package,
   182  	"import":      Import,
   183  	"func":        Func,
   184  	"return":      Return,
   185  	"select":      Select,
   186  	"switch":      Switch,
   187  	"case":        Case,
   188  	"default":     Default,
   189  	"defer":       Defer,
   190  	"fallthrough": Fallthrough,
   191  	"const":       Const,
   192  	"var":         Var,
   193  	"if":          If,
   194  	"else":        Else,
   195  	"for":         For,
   196  	"range":       Range,
   197  	"continue":    Continue,
   198  	"break":       Break,
   199  	"goto":        Goto,
   200  	"go":          Go,
   201  	"chan":        Chan,
   202  	"map":         Map,
   203  	"struct":      Struct,
   204  	"methodik":    Methodik,
   205  	"interface":   Interface,
   206  	"type":        Type,
   207  }
   208  
   209  func Keyword(n string) Token {
   210  	return Keywords[n]
   211  }
   212  
   213  var tokenStrings = make(map[Token]string, len(tokens)+len(Keywords))
   214  
   215  func init() {
   216  	for s, t := range tokens {
   217  		tokenStrings[t] = s
   218  	}
   219  	for s, t := range Keywords {
   220  		tokenStrings[t] = s
   221  	}
   222  }
   223  
   224  func (t Token) String() string {
   225  	if s := tokenStrings[t]; s != "" {
   226  		return s
   227  	}
   228  	return fmt.Sprintf("Token:%d", t)
   229  }
   230  
   231  func (t Token) Precedence() int {
   232  	// see:
   233  	// https://golang.org/ref/spec#Operator_precedence
   234  	switch t {
   235  	case LogicalOr:
   236  		return 1
   237  	case LogicalAnd:
   238  		return 2
   239  	case Equal, NotEqual, Less, LessEqual, Greater, GreaterEqual:
   240  		return 3
   241  	case Add, Sub, Pipe, Pow:
   242  		return 4
   243  	case Mul, Div, Ref, Rem, TwoLess, RefPow, TwoGreater:
   244  		return 5
   245  	}
   246  	return 0
   247  }