github.com/quantosnetwork/Quantos@v0.0.0-20220306172517-e20b28c5a29a/quantix/interpreter/lexer.go (about) 1 package interpreter 2 3 import ( 4 "github.com/quantosnetwork/Quantos/quantix/token" 5 "io/ioutil" 6 "unicode" 7 ) 8 9 type state int 10 11 const nullState state = -1 12 13 type Lexer struct { 14 I []rune 15 Tokens []*token.Token 16 } 17 18 func (l *Lexer) scan(i int) *token.Token { 19 s, typ, rext := nullState, token.Error, i+1 20 if i < len(l.I) { 21 s = nextState[0](l.I[i]) 22 } 23 for s != nullState { 24 if rext >= len(l.I) { 25 typ = accept[s] 26 s = nullState 27 } else { 28 typ = accept[s] 29 s = nextState[s](l.I[rext]) 30 if s != nullState || typ == token.Error { 31 rext++ 32 } 33 } 34 } 35 tok := token.NewToken(typ, i, rext, l.I) 36 return tok 37 } 38 39 func InitNewLexer(_src []rune) *Lexer { 40 lex := new(Lexer) 41 lex.I = _src 42 lex.Tokens = make([]*token.Token, 0, 2048) 43 44 lext := 0 45 for lext < len(lex.I) { 46 for lext < len(lex.I) && unicode.IsSpace(lex.I[lext]) { 47 lext++ 48 } 49 if lext < len(lex.I) { 50 tok := lex.scan(lext) 51 lext = tok.Rext() 52 if !tok.Suppress() { 53 } 54 lex.addToken(tok) 55 } 56 } 57 58 lex.add(token.EOF, len(_src), len(_src)) 59 60 return lex 61 } 62 63 func NewInputFile(fname string) *Lexer { 64 buf, err := ioutil.ReadFile(fname) 65 if err != nil { 66 panic(err) 67 } 68 69 return InitNewLexer([]rune(string(buf))) 70 } 71 72 // util functions 73 74 func isWhitespace(r rune) bool { 75 return r == ' ' || r == '\n' || r == '\t' 76 } 77 78 func isLower(r rune) bool { 79 return 'a' <= r && r <= 'z' 80 } 81 82 func isUpper(r rune) bool { 83 return 'A' <= r && r <= 'Z' 84 } 85 86 func isLetter(r rune) bool { 87 return isLower(r) || isUpper(r) 88 } 89 90 func isIdentStartChar(r rune) bool { 91 return r == '_' || isLetter(r) 92 } 93 94 func isDigit(r rune) bool { 95 return '0' <= r && r <= '9' 96 } 97 98 func isIdentChar(r rune) bool { 99 return isIdentStartChar(r) || isDigit(r) 100 } 101 102 func escape(r rune) string { 103 switch r { 104 case '"': 105 return "\"" 106 case '\\': 107 return "\\\\" 108 case '\r': 109 return "\\r" 110 case '\n': 111 return "\\n" 112 case '\t': 113 return "\\t" 114 } 115 return string(r) 116 } 117 118 // GetLineColumn returns the line and column of rune[i] in the input 119 func (l *Lexer) GetLineColumn(i int) (line, col int) { 120 line, col = 1, 1 121 for j := 0; j < i; j++ { 122 switch l.I[j] { 123 case '\n': 124 line++ 125 col = 1 126 case '\t': 127 col += 4 128 default: 129 col++ 130 } 131 } 132 return 133 } 134 135 // GetLineColumnOfToken returns the line and column of token[i] in the imput 136 func (l *Lexer) GetLineColumnOfToken(i int) (line, col int) { 137 return l.GetLineColumn(l.Tokens[i].Lext()) 138 } 139 140 // GetString returns the input string from the left extent of Token[lext] to 141 // the right extent of Token[rext] 142 func (l *Lexer) GetString(lext, rext int) string { 143 return string(l.I[l.Tokens[lext].Lext():l.Tokens[rext].Rext()]) 144 } 145 146 func (l *Lexer) add(t token.Type, lext, rext int) { 147 l.addToken(token.NewToken(token.TokenType(t), lext, rext, l.I)) 148 } 149 150 func (l *Lexer) addToken(tok *token.Token) { 151 l.Tokens = append(l.Tokens, tok) 152 } 153 154 func any(r rune, set []rune) bool { 155 for _, r1 := range set { 156 if r == r1 { 157 return true 158 } 159 } 160 return false 161 } 162 163 func not(r rune, set []rune) bool { 164 for _, r1 := range set { 165 if r == r1 { 166 return false 167 } 168 } 169 return true 170 } 171 172 var accept = []token.Type{ 173 token.Error, 174 token.T_0, 175 token.Error, 176 token.Error, 177 token.T_1, 178 token.T_2, 179 token.T_3, 180 token.T_4, 181 token.T_5, 182 token.T_6, 183 token.T_7, 184 token.T_8, 185 token.T_9, 186 token.Error, 187 token.Error, 188 token.Error, 189 token.Error, 190 token.Error, 191 token.Error, 192 token.T_22, 193 token.T_23, 194 token.T_24, 195 token.Error, 196 token.Error, 197 token.Error, 198 token.Error, 199 token.Error, 200 token.Error, 201 token.T_20, 202 token.T_20, 203 token.T_20, 204 token.T_20, 205 token.T_20, 206 token.T_20, 207 token.T_20, 208 token.T_20, 209 token.T_20, 210 token.T_16, 211 token.T_19, 212 token.T_11, 213 token.T_11, 214 token.T_10, 215 token.T_20, 216 token.T_20, 217 token.T_20, 218 token.T_15, 219 token.T_20, 220 token.T_20, 221 token.T_20, 222 token.T_20, 223 token.T_20, 224 token.T_20, 225 token.T_20, 226 token.T_20, 227 token.T_20, 228 token.T_12, 229 token.T_20, 230 token.T_20, 231 token.T_20, 232 token.T_20, 233 token.T_20, 234 token.T_13, 235 token.T_20, 236 token.T_17, 237 token.T_20, 238 token.T_21, 239 token.T_14, 240 token.T_18, 241 } 242 243 var nextState = []func(r rune) state{ 244 // Set0 245 func(r rune) state { 246 switch { 247 case r == '!': 248 return 1 249 case r == '"': 250 return 2 251 case r == '\'': 252 return 3 253 case r == '(': 254 return 4 255 case r == ')': 256 return 5 257 case r == '.': 258 return 6 259 case r == ':': 260 return 7 261 case r == ';': 262 return 8 263 case r == '<': 264 return 9 265 case r == '>': 266 return 10 267 case r == '[': 268 return 11 269 case r == ']': 270 return 12 271 case r == 'a': 272 return 13 273 case r == 'e': 274 return 14 275 case r == 'l': 276 return 15 277 case r == 'n': 278 return 16 279 case r == 'p': 280 return 17 281 case r == 'u': 282 return 18 283 case r == '{': 284 return 19 285 case r == '|': 286 return 20 287 case r == '}': 288 return 21 289 case unicode.IsUpper(r): 290 return 22 291 case unicode.IsLower(r): 292 return 23 293 } 294 return nullState 295 }, 296 // Set1 297 func(r rune) state { 298 switch { 299 } 300 return nullState 301 }, 302 // Set2 303 func(r rune) state { 304 switch { 305 case r == '\\': 306 return 24 307 case not(r, []rune{'"', '\\'}): 308 return 25 309 } 310 return nullState 311 }, 312 // Set3 313 func(r rune) state { 314 switch { 315 case r == '\\': 316 return 26 317 case not(r, []rune{'\''}): 318 return 27 319 } 320 return nullState 321 }, 322 // Set4 323 func(r rune) state { 324 switch { 325 } 326 return nullState 327 }, 328 // Set5 329 func(r rune) state { 330 switch { 331 } 332 return nullState 333 }, 334 // Set6 335 func(r rune) state { 336 switch { 337 } 338 return nullState 339 }, 340 // Set7 341 func(r rune) state { 342 switch { 343 } 344 return nullState 345 }, 346 // Set8 347 func(r rune) state { 348 switch { 349 } 350 return nullState 351 }, 352 // Set9 353 func(r rune) state { 354 switch { 355 } 356 return nullState 357 }, 358 // Set10 359 func(r rune) state { 360 switch { 361 } 362 return nullState 363 }, 364 // Set11 365 func(r rune) state { 366 switch { 367 } 368 return nullState 369 }, 370 // Set12 371 func(r rune) state { 372 switch { 373 } 374 return nullState 375 }, 376 // Set13 377 func(r rune) state { 378 switch { 379 case r == '_': 380 return 28 381 case r == 'n': 382 return 29 383 case unicode.IsLetter(r): 384 return 28 385 case unicode.IsNumber(r): 386 return 28 387 } 388 return nullState 389 }, 390 // Set14 391 func(r rune) state { 392 switch { 393 case r == '_': 394 return 28 395 case r == 'm': 396 return 30 397 case unicode.IsLetter(r): 398 return 28 399 case unicode.IsNumber(r): 400 return 28 401 } 402 return nullState 403 }, 404 // Set15 405 func(r rune) state { 406 switch { 407 case r == '_': 408 return 28 409 case r == 'e': 410 return 31 411 case r == 'o': 412 return 32 413 case unicode.IsLetter(r): 414 return 28 415 case unicode.IsNumber(r): 416 return 28 417 } 418 return nullState 419 }, 420 // Set16 421 func(r rune) state { 422 switch { 423 case r == '_': 424 return 28 425 case r == 'o': 426 return 33 427 case r == 'u': 428 return 34 429 case unicode.IsLetter(r): 430 return 28 431 case unicode.IsNumber(r): 432 return 28 433 } 434 return nullState 435 }, 436 // Set17 437 func(r rune) state { 438 switch { 439 case r == '_': 440 return 28 441 case r == 'a': 442 return 35 443 case unicode.IsLetter(r): 444 return 28 445 case unicode.IsNumber(r): 446 return 28 447 } 448 return nullState 449 }, 450 // Set18 451 func(r rune) state { 452 switch { 453 case r == '_': 454 return 28 455 case r == 'p': 456 return 36 457 case unicode.IsLetter(r): 458 return 28 459 case unicode.IsNumber(r): 460 return 28 461 } 462 return nullState 463 }, 464 // Set19 465 func(r rune) state { 466 switch { 467 } 468 return nullState 469 }, 470 // Set20 471 func(r rune) state { 472 switch { 473 } 474 return nullState 475 }, 476 // Set21 477 func(r rune) state { 478 switch { 479 } 480 return nullState 481 }, 482 // Set22 483 func(r rune) state { 484 switch { 485 case r == '_': 486 return 37 487 case unicode.IsLetter(r): 488 return 37 489 case unicode.IsNumber(r): 490 return 37 491 } 492 return nullState 493 }, 494 // Set23 495 func(r rune) state { 496 switch { 497 case r == '_': 498 return 28 499 case unicode.IsLetter(r): 500 return 28 501 case unicode.IsNumber(r): 502 return 28 503 } 504 return nullState 505 }, 506 // Set24 507 func(r rune) state { 508 switch { 509 case any(r, []rune{'"', '\\', 'n', 'r', 't'}): 510 return 25 511 } 512 return nullState 513 }, 514 // Set25 515 func(r rune) state { 516 switch { 517 case r == '"': 518 return 38 519 case r == '\\': 520 return 24 521 case not(r, []rune{'"', '\\'}): 522 return 25 523 } 524 return nullState 525 }, 526 // Set26 527 func(r rune) state { 528 switch { 529 case any(r, []rune{'\'', '\\', 'n', 'r', 't'}): 530 return 39 531 case r == '\'': 532 return 39 533 } 534 return nullState 535 }, 536 // Set27 537 func(r rune) state { 538 switch { 539 case r == '\'': 540 return 40 541 } 542 return nullState 543 }, 544 // Set28 545 func(r rune) state { 546 switch { 547 case r == '_': 548 return 28 549 case unicode.IsLetter(r): 550 return 28 551 case unicode.IsNumber(r): 552 return 28 553 } 554 return nullState 555 }, 556 // Set29 557 func(r rune) state { 558 switch { 559 case r == '_': 560 return 28 561 case r == 'y': 562 return 41 563 case unicode.IsLetter(r): 564 return 28 565 case unicode.IsNumber(r): 566 return 28 567 } 568 return nullState 569 }, 570 // Set30 571 func(r rune) state { 572 switch { 573 case r == '_': 574 return 28 575 case r == 'p': 576 return 42 577 case unicode.IsLetter(r): 578 return 28 579 case unicode.IsNumber(r): 580 return 28 581 } 582 return nullState 583 }, 584 // Set31 585 func(r rune) state { 586 switch { 587 case r == '_': 588 return 28 589 case r == 't': 590 return 43 591 case unicode.IsLetter(r): 592 return 28 593 case unicode.IsNumber(r): 594 return 28 595 } 596 return nullState 597 }, 598 // Set32 599 func(r rune) state { 600 switch { 601 case r == '_': 602 return 28 603 case r == 'w': 604 return 44 605 case unicode.IsLetter(r): 606 return 28 607 case unicode.IsNumber(r): 608 return 28 609 } 610 return nullState 611 }, 612 // Set33 613 func(r rune) state { 614 switch { 615 case r == '_': 616 return 28 617 case r == 't': 618 return 45 619 case unicode.IsLetter(r): 620 return 28 621 case unicode.IsNumber(r): 622 return 28 623 } 624 return nullState 625 }, 626 // Set34 627 func(r rune) state { 628 switch { 629 case r == '_': 630 return 28 631 case r == 'm': 632 return 46 633 case unicode.IsLetter(r): 634 return 28 635 case unicode.IsNumber(r): 636 return 28 637 } 638 return nullState 639 }, 640 // Set35 641 func(r rune) state { 642 switch { 643 case r == '_': 644 return 28 645 case r == 'c': 646 return 47 647 case unicode.IsLetter(r): 648 return 28 649 case unicode.IsNumber(r): 650 return 28 651 } 652 return nullState 653 }, 654 // Set36 655 func(r rune) state { 656 switch { 657 case r == '_': 658 return 28 659 case r == 'c': 660 return 48 661 case unicode.IsLetter(r): 662 return 28 663 case unicode.IsNumber(r): 664 return 28 665 } 666 return nullState 667 }, 668 // Set37 669 func(r rune) state { 670 switch { 671 case r == '_': 672 return 37 673 case unicode.IsLetter(r): 674 return 37 675 case unicode.IsNumber(r): 676 return 37 677 } 678 return nullState 679 }, 680 // Set38 681 func(r rune) state { 682 switch { 683 } 684 return nullState 685 }, 686 // Set39 687 func(r rune) state { 688 switch { 689 case r == '\'': 690 return 40 691 } 692 return nullState 693 }, 694 // Set40 695 func(r rune) state { 696 switch { 697 } 698 return nullState 699 }, 700 // Set41 701 func(r rune) state { 702 switch { 703 case r == '_': 704 return 28 705 case unicode.IsLetter(r): 706 return 28 707 case unicode.IsNumber(r): 708 return 28 709 } 710 return nullState 711 }, 712 // Set42 713 func(r rune) state { 714 switch { 715 case r == '_': 716 return 28 717 case r == 't': 718 return 49 719 case unicode.IsLetter(r): 720 return 28 721 case unicode.IsNumber(r): 722 return 28 723 } 724 return nullState 725 }, 726 // Set43 727 func(r rune) state { 728 switch { 729 case r == '_': 730 return 28 731 case r == 't': 732 return 50 733 case unicode.IsLetter(r): 734 return 28 735 case unicode.IsNumber(r): 736 return 28 737 } 738 return nullState 739 }, 740 // Set44 741 func(r rune) state { 742 switch { 743 case r == '_': 744 return 28 745 case r == 'c': 746 return 51 747 case unicode.IsLetter(r): 748 return 28 749 case unicode.IsNumber(r): 750 return 28 751 } 752 return nullState 753 }, 754 // Set45 755 func(r rune) state { 756 switch { 757 case r == '_': 758 return 28 759 case unicode.IsLetter(r): 760 return 28 761 case unicode.IsNumber(r): 762 return 28 763 } 764 return nullState 765 }, 766 // Set46 767 func(r rune) state { 768 switch { 769 case r == '_': 770 return 28 771 case r == 'b': 772 return 52 773 case unicode.IsLetter(r): 774 return 28 775 case unicode.IsNumber(r): 776 return 28 777 } 778 return nullState 779 }, 780 // Set47 781 func(r rune) state { 782 switch { 783 case r == '_': 784 return 28 785 case r == 'k': 786 return 53 787 case unicode.IsLetter(r): 788 return 28 789 case unicode.IsNumber(r): 790 return 28 791 } 792 return nullState 793 }, 794 // Set48 795 func(r rune) state { 796 switch { 797 case r == '_': 798 return 28 799 case r == 'a': 800 return 54 801 case unicode.IsLetter(r): 802 return 28 803 case unicode.IsNumber(r): 804 return 28 805 } 806 return nullState 807 }, 808 // Set49 809 func(r rune) state { 810 switch { 811 case r == '_': 812 return 28 813 case r == 'y': 814 return 55 815 case unicode.IsLetter(r): 816 return 28 817 case unicode.IsNumber(r): 818 return 28 819 } 820 return nullState 821 }, 822 // Set50 823 func(r rune) state { 824 switch { 825 case r == '_': 826 return 28 827 case r == 'e': 828 return 56 829 case unicode.IsLetter(r): 830 return 28 831 case unicode.IsNumber(r): 832 return 28 833 } 834 return nullState 835 }, 836 // Set51 837 func(r rune) state { 838 switch { 839 case r == '_': 840 return 28 841 case r == 'a': 842 return 57 843 case unicode.IsLetter(r): 844 return 28 845 case unicode.IsNumber(r): 846 return 28 847 } 848 return nullState 849 }, 850 // Set52 851 func(r rune) state { 852 switch { 853 case r == '_': 854 return 28 855 case r == 'e': 856 return 58 857 case unicode.IsLetter(r): 858 return 28 859 case unicode.IsNumber(r): 860 return 28 861 } 862 return nullState 863 }, 864 // Set53 865 func(r rune) state { 866 switch { 867 case r == '_': 868 return 28 869 case r == 'a': 870 return 59 871 case unicode.IsLetter(r): 872 return 28 873 case unicode.IsNumber(r): 874 return 28 875 } 876 return nullState 877 }, 878 // Set54 879 func(r rune) state { 880 switch { 881 case r == '_': 882 return 28 883 case r == 's': 884 return 60 885 case unicode.IsLetter(r): 886 return 28 887 case unicode.IsNumber(r): 888 return 28 889 } 890 return nullState 891 }, 892 // Set55 893 func(r rune) state { 894 switch { 895 case r == '_': 896 return 28 897 case unicode.IsLetter(r): 898 return 28 899 case unicode.IsNumber(r): 900 return 28 901 } 902 return nullState 903 }, 904 // Set56 905 func(r rune) state { 906 switch { 907 case r == '_': 908 return 28 909 case r == 'r': 910 return 61 911 case unicode.IsLetter(r): 912 return 28 913 case unicode.IsNumber(r): 914 return 28 915 } 916 return nullState 917 }, 918 // Set57 919 func(r rune) state { 920 switch { 921 case r == '_': 922 return 28 923 case r == 's': 924 return 62 925 case unicode.IsLetter(r): 926 return 28 927 case unicode.IsNumber(r): 928 return 28 929 } 930 return nullState 931 }, 932 // Set58 933 func(r rune) state { 934 switch { 935 case r == '_': 936 return 28 937 case r == 'r': 938 return 63 939 case unicode.IsLetter(r): 940 return 28 941 case unicode.IsNumber(r): 942 return 28 943 } 944 return nullState 945 }, 946 // Set59 947 func(r rune) state { 948 switch { 949 case r == '_': 950 return 28 951 case r == 'g': 952 return 64 953 case unicode.IsLetter(r): 954 return 28 955 case unicode.IsNumber(r): 956 return 28 957 } 958 return nullState 959 }, 960 // Set60 961 func(r rune) state { 962 switch { 963 case r == '_': 964 return 28 965 case r == 'e': 966 return 65 967 case unicode.IsLetter(r): 968 return 28 969 case unicode.IsNumber(r): 970 return 28 971 } 972 return nullState 973 }, 974 // Set61 975 func(r rune) state { 976 switch { 977 case r == '_': 978 return 28 979 case unicode.IsLetter(r): 980 return 28 981 case unicode.IsNumber(r): 982 return 28 983 } 984 return nullState 985 }, 986 // Set62 987 func(r rune) state { 988 switch { 989 case r == '_': 990 return 28 991 case r == 'e': 992 return 66 993 case unicode.IsLetter(r): 994 return 28 995 case unicode.IsNumber(r): 996 return 28 997 } 998 return nullState 999 }, 1000 // Set63 1001 func(r rune) state { 1002 switch { 1003 case r == '_': 1004 return 28 1005 case unicode.IsLetter(r): 1006 return 28 1007 case unicode.IsNumber(r): 1008 return 28 1009 } 1010 return nullState 1011 }, 1012 // Set64 1013 func(r rune) state { 1014 switch { 1015 case r == '_': 1016 return 28 1017 case r == 'e': 1018 return 67 1019 case unicode.IsLetter(r): 1020 return 28 1021 case unicode.IsNumber(r): 1022 return 28 1023 } 1024 return nullState 1025 }, 1026 // Set65 1027 func(r rune) state { 1028 switch { 1029 case r == '_': 1030 return 28 1031 case unicode.IsLetter(r): 1032 return 28 1033 case unicode.IsNumber(r): 1034 return 28 1035 } 1036 return nullState 1037 }, 1038 // Set66 1039 func(r rune) state { 1040 switch { 1041 case r == '_': 1042 return 28 1043 case unicode.IsLetter(r): 1044 return 28 1045 case unicode.IsNumber(r): 1046 return 28 1047 } 1048 return nullState 1049 }, 1050 // Set67 1051 func(r rune) state { 1052 switch { 1053 case r == '_': 1054 return 28 1055 case unicode.IsLetter(r): 1056 return 28 1057 case unicode.IsNumber(r): 1058 return 28 1059 } 1060 return nullState 1061 }, 1062 }