git.templeos.me/xultist/go-enry/v2@v2.0.0-20230215093429-6ef3e87f47c0/internal/tokenizer/flex/linguist.h (about) 1 // https://github.com/github/linguist/blob/f72f2a21dfe80ebd16af3bc6216da75cd983a4f6/ext/linguist/linguist.h#L1 2 enum tokenizer_type { 3 NO_ACTION, 4 REGULAR_TOKEN, 5 SHEBANG_TOKEN, 6 SGML_TOKEN, 7 }; 8 9 struct tokenizer_extra { 10 char *token; 11 enum tokenizer_type type; 12 }; 13 14 // TODO(bzz) port Win support from 15 // https://github.com/github/linguist/commit/8e912b4d8bf2aef7948de59eba48b75cfcbc97e0