github.com/joomcode/cue@v0.4.4-0.20221111115225-539fe3512047/cue/token/token.go (about) 1 // Copyright 2018 The CUE Authors 2 // 3 // Licensed under the Apache License, Version 2.0 (the "License"); 4 // you may not use this file except in compliance with the License. 5 // You may obtain a copy of the License at 6 // 7 // http://www.apache.org/licenses/LICENSE-2.0 8 // 9 // Unless required by applicable law or agreed to in writing, software 10 // distributed under the License is distributed on an "AS IS" BASIS, 11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 // See the License for the specific language governing permissions and 13 // limitations under the License. 14 15 // Package token defines constants representing the lexical tokens of the Go 16 // programming language and basic operations on tokens (printing, predicates). 17 package token // import "github.com/joomcode/cue/cue/token" 18 19 import "strconv" 20 21 // Token is the set of lexical tokens of the CUE configuration language. 22 type Token int 23 24 // The list of tokens. 25 const ( 26 // Special tokens 27 ILLEGAL Token = iota 28 EOF 29 COMMENT 30 ATTRIBUTE // @foo(bar,baz=4) 31 32 literalBeg 33 // Identifiers and basic type literals 34 // (these tokens stand for classes of literals) 35 IDENT // main, _tmp 36 INT // 12_345Mi, 0700, 0xdeadbeef, 1.2M 37 FLOAT // 123.45, 38 // DURATION // 3m4s TODO 39 STRING // "abc" 40 INTERPOLATION // a part of a template string, e.g. `"age: \(` 41 BOTTOM // _|_ 42 43 literalEnd 44 45 operatorBeg 46 // Operators and delimiters 47 ADD // + 48 SUB // - 49 MUL // * 50 POW // ^ 51 QUO // / 52 53 IQUO // quo 54 IREM // rem 55 IDIV // div 56 IMOD // mod 57 58 AND // & 59 OR // | 60 61 LAND // && 62 LOR // || 63 64 BIND // = 65 EQL // == 66 LSS // < 67 GTR // > 68 NOT // ! 69 ARROW // <- 70 71 NEQ // != 72 LEQ // <= 73 GEQ // >= 74 75 MAT // =~ 76 NMAT // !~ 77 78 LPAREN // ( 79 LBRACK // [ 80 LBRACE // { 81 COMMA // , 82 PERIOD // . 83 ELLIPSIS // ... 84 85 RPAREN // ) 86 RBRACK // ] 87 RBRACE // } 88 SEMICOLON // ; 89 COLON // : 90 ISA // :: 91 OPTION // ? 92 operatorEnd 93 94 keywordBeg 95 96 IF 97 FOR 98 IN 99 LET 100 101 TRUE 102 FALSE 103 NULL 104 105 keywordEnd 106 ) 107 108 var tokens = [...]string{ 109 ILLEGAL: "ILLEGAL", 110 111 EOF: "EOF", 112 COMMENT: "COMMENT", 113 114 IDENT: "IDENT", 115 INT: "INT", 116 FLOAT: "FLOAT", 117 STRING: "STRING", 118 INTERPOLATION: "INTERPOLATION", 119 ATTRIBUTE: "ATTRIBUTE", 120 121 ADD: "+", 122 SUB: "-", 123 MUL: "*", 124 POW: "^", 125 QUO: "/", 126 127 IQUO: "quo", 128 IREM: "rem", 129 IDIV: "div", 130 IMOD: "mod", 131 132 AND: "&", 133 OR: "|", 134 135 LAND: "&&", 136 LOR: "||", 137 138 BIND: "=", 139 EQL: "==", 140 LSS: "<", 141 GTR: ">", 142 NOT: "!", 143 ARROW: "<-", 144 145 NEQ: "!=", 146 LEQ: "<=", 147 GEQ: ">=", 148 149 MAT: "=~", 150 NMAT: "!~", 151 152 LPAREN: "(", 153 LBRACK: "[", 154 LBRACE: "{", 155 COMMA: ",", 156 PERIOD: ".", 157 ELLIPSIS: "...", 158 159 RPAREN: ")", 160 RBRACK: "]", 161 RBRACE: "}", 162 SEMICOLON: ";", 163 COLON: ":", 164 ISA: "::", 165 OPTION: "?", 166 167 BOTTOM: "_|_", 168 169 FALSE: "false", 170 TRUE: "true", 171 NULL: "null", 172 173 FOR: "for", 174 IF: "if", 175 IN: "in", 176 LET: "let", 177 } 178 179 // String returns the string corresponding to the token tok. 180 // For operators, delimiters, and keywords the string is the actual 181 // token character sequence (e.g., for the token ADD, the string is 182 // "+"). For all other tokens the string corresponds to the token 183 // constant name (e.g. for the token IDENT, the string is "IDENT"). 184 func (tok Token) String() string { 185 s := "" 186 if 0 <= tok && tok < Token(len(tokens)) { 187 s = tokens[tok] 188 } 189 if s == "" { 190 s = "token(" + strconv.Itoa(int(tok)) + ")" 191 } 192 return s 193 } 194 195 // A set of constants for precedence-based expression parsing. 196 // Non-operators have lowest precedence, followed by operators 197 // starting with precedence 1 up to unary operators. The highest 198 // precedence serves as "catch-all" precedence for selector, 199 // indexing, and other operator and delimiter tokens. 200 const ( 201 LowestPrec = lowestPrec 202 UnaryPrec = unaryPrec 203 HighestPrec = highestPrec 204 ) 205 206 const ( 207 lowestPrec = 0 // non-operators 208 unaryPrec = 8 209 highestPrec = 9 210 ) 211 212 // Precedence returns the operator precedence of the binary 213 // operator op. If op is not a binary operator, the result 214 // is LowestPrecedence. 215 // 216 func (tok Token) Precedence() int { 217 switch tok { 218 case OR: 219 return 1 220 case AND: 221 return 2 222 case LOR: 223 return 3 224 case LAND: 225 return 4 226 case EQL, NEQ, LSS, LEQ, GTR, GEQ, MAT, NMAT: 227 return 5 228 case ADD, SUB: 229 return 6 230 case MUL, QUO, IDIV, IMOD, IQUO, IREM: 231 return 7 232 } 233 return lowestPrec 234 } 235 236 var keywords map[string]Token 237 238 func init() { 239 keywords = make(map[string]Token) 240 for i := keywordBeg + 1; i < keywordEnd; i++ { 241 keywords[tokens[i]] = i 242 } 243 } 244 245 // Lookup maps an identifier to its keyword token or IDENT (if not a keyword). 246 // 247 func Lookup(ident string) Token { 248 if tok, isKeyword := keywords[ident]; isKeyword { 249 return tok 250 } 251 return IDENT 252 } 253 254 // Predicates 255 256 // IsLiteral returns true for tokens corresponding to identifiers 257 // and basic type literals; it returns false otherwise. 258 func (tok Token) IsLiteral() bool { return literalBeg < tok && tok < literalEnd } 259 260 // IsOperator returns true for tokens corresponding to operators and 261 // delimiters; it returns false otherwise. 262 func (tok Token) IsOperator() bool { return operatorBeg < tok && tok < operatorEnd } 263 264 // IsKeyword returns true for tokens corresponding to keywords; 265 // it returns false otherwise. 266 func (tok Token) IsKeyword() bool { return keywordBeg < tok && tok < keywordEnd }