github.com/nuvolaris/goja@v0.0.0-20230825100449-967811910c6d/token/tokenfmt (about)

     1  #!/usr/bin/env perl
     2  
     3  use strict;
     4  use warnings;
     5  
     6  my (%token, @order, @keywords);
     7  
     8  {
     9      my $keywords;
    10      my @const;
    11      push @const, <<_END_;
    12  package token
    13  
    14  const(
    15      _ Token = iota
    16  _END_
    17  
    18      for (split m/\n/, <<_END_) {
    19  ILLEGAL
    20  EOF
    21  COMMENT
    22  KEYWORD
    23  
    24  STRING
    25  BOOLEAN
    26  NULL
    27  NUMBER
    28  IDENTIFIER
    29  
    30  PLUS                            +
    31  MINUS                           -
    32  MULTIPLY                        *
    33  SLASH                           /
    34  REMAINDER                       %
    35  
    36  AND                             &
    37  OR                              |
    38  EXCLUSIVE_OR                    ^
    39  SHIFT_LEFT                      <<
    40  SHIFT_RIGHT                     >>
    41  UNSIGNED_SHIFT_RIGHT            >>>
    42  AND_NOT                         &^
    43  
    44  ADD_ASSIGN                     +=
    45  SUBTRACT_ASSIGN                -=
    46  MULTIPLY_ASSIGN                *=
    47  QUOTIENT_ASSIGN                /=
    48  REMAINDER_ASSIGN               %=
    49  
    50  AND_ASSIGN                     &=
    51  OR_ASSIGN                      |=
    52  EXCLUSIVE_OR_ASSIGN            ^=
    53  SHIFT_LEFT_ASSIGN              <<=
    54  SHIFT_RIGHT_ASSIGN             >>=
    55  UNSIGNED_SHIFT_RIGHT_ASSIGN    >>>=
    56  AND_NOT_ASSIGN                 &^=
    57  
    58  LOGICAL_AND                    &&
    59  LOGICAL_OR                     ||
    60  INCREMENT                      ++
    61  DECREMENT                      --
    62  
    63  EQUAL                          ==
    64  STRICT_EQUAL                   ===
    65  LESS                           <
    66  GREATER                        >
    67  ASSIGN                         =
    68  NOT                            !
    69  
    70  BITWISE_NOT                    ~
    71  
    72  NOT_EQUAL                      !=
    73  STRICT_NOT_EQUAL               !==
    74  LESS_OR_EQUAL                  <=
    75  GREATER_OR_EQUAL               <=
    76  
    77  LEFT_PARENTHESIS               (
    78  LEFT_BRACKET                   [
    79  LEFT_BRACE                     {
    80  COMMA                          ,
    81  PERIOD                         .
    82  
    83  RIGHT_PARENTHESIS              )
    84  RIGHT_BRACKET                  ]
    85  RIGHT_BRACE                    }
    86  SEMICOLON                      ;
    87  COLON                          :
    88  QUESTION_MARK                  ?
    89  
    90  firstKeyword
    91  IF
    92  IN
    93  DO
    94  
    95  VAR
    96  FOR
    97  NEW
    98  TRY
    99  
   100  THIS
   101  ELSE
   102  CASE
   103  VOID
   104  WITH
   105  
   106  WHILE
   107  BREAK
   108  CATCH
   109  THROW
   110  
   111  RETURN
   112  TYPEOF
   113  DELETE
   114  SWITCH
   115  
   116  DEFAULT
   117  FINALLY
   118  
   119  FUNCTION
   120  CONTINUE
   121  DEBUGGER
   122  
   123  INSTANCEOF
   124  lastKeyword
   125  _END_
   126          chomp;
   127  
   128          next if m/^\s*#/;
   129  
   130          my ($name, $symbol) = m/(\w+)\s*(\S+)?/;
   131  
   132          if (defined $symbol) {
   133              push @order, $name;
   134              push @const, "$name // $symbol";
   135              $token{$name} = $symbol;
   136          } elsif (defined $name) {
   137              $keywords ||= $name eq 'firstKeyword';
   138  
   139              push @const, $name;
   140              #$const[-1] .= " Token = iota" if 2 == @const;
   141              if ($name =~ m/^([A-Z]+)/) {
   142                  push @keywords, $name if $keywords;
   143                  push @order, $name;
   144                  if ($token{SEMICOLON}) {
   145                      $token{$name} = lc $1;
   146                  } else {
   147                      $token{$name} = $name;
   148                  }
   149              }
   150          } else {
   151              push @const, "";
   152          }
   153  
   154      }
   155      push @const, ")";
   156      print join "\n", @const, "";
   157  }
   158  
   159  {
   160      print <<_END_;
   161  
   162  var token2string = [...]string{
   163  _END_
   164      for my $name (@order) {
   165          print "$name: \"$token{$name}\",\n";
   166      }
   167      print <<_END_;
   168  }
   169  _END_
   170  
   171      print <<_END_;
   172  
   173  var keywordTable = map[string]_keyword{
   174  _END_
   175      for my $name (@keywords) {
   176          print <<_END_
   177  			"@{[ lc $name ]}": _keyword{
   178  				token: $name,
   179  			},
   180  _END_
   181      }
   182  
   183      for my $name (qw/
   184          const
   185          class
   186          enum
   187          export
   188          extends
   189          import
   190          super
   191          /) {
   192          print <<_END_
   193  			"$name": _keyword{
   194  				token: KEYWORD,
   195                  futureKeyword: true,
   196  			},
   197  _END_
   198      }
   199  
   200      for my $name (qw/
   201          implements
   202          interface
   203          let
   204          package
   205          private
   206          protected
   207          public
   208          static
   209          /) {
   210          print <<_END_
   211  			"$name": _keyword{
   212  				token: KEYWORD,
   213                  futureKeyword: true,
   214                  strict: true,
   215  			},
   216  _END_
   217      }
   218  
   219      print <<_END_;
   220  }
   221  _END_
   222  }