git.templeos.me/xultist/go-enry/v2@v2.0.0-20230215093429-6ef3e87f47c0/internal/tokenizer/flex/linguist.h (about)

     1  // https://github.com/github/linguist/blob/f72f2a21dfe80ebd16af3bc6216da75cd983a4f6/ext/linguist/linguist.h#L1
     2  enum tokenizer_type {
     3    NO_ACTION,
     4    REGULAR_TOKEN,
     5    SHEBANG_TOKEN,
     6    SGML_TOKEN,
     7  };
     8  
     9  struct tokenizer_extra {
    10    char *token;
    11    enum tokenizer_type type;
    12  };
    13  
    14  // TODO(bzz) port Win support from
    15  // https://github.com/github/linguist/commit/8e912b4d8bf2aef7948de59eba48b75cfcbc97e0