github.com/wbrown/gpt_bpe@v0.0.0-20250709161131-1571a6e8ad2d/lib/library.h (about)

     1  #ifndef GPT_BPE_LIB_H
     2  #define GPT_BPE_LIB_H
     3  
     4  #include <stdlib.h>
     5  #include <stdint.h>
     6  
     7  typedef struct {
     8  	uint32_t *tokens;
     9  	size_t len;
    10  } Tokens;
    11  
    12  #endif