github.com/wbrown/gpt_bpe@v0.0.0-20250709161131-1571a6e8ad2d/lib/gpt_bpe.h (about)

     1  /* Code generated by cmd/cgo; DO NOT EDIT. */
     2  
     3  /* package command-line-arguments */
     4  
     5  
     6  #line 1 "cgo-builtin-export-prolog"
     7  
     8  #include <stddef.h> /* for ptrdiff_t below */
     9  
    10  #ifndef GO_CGO_EXPORT_PROLOGUE_H
    11  #define GO_CGO_EXPORT_PROLOGUE_H
    12  
    13  #ifndef GO_CGO_GOSTRING_TYPEDEF
    14  typedef struct { const char *p; ptrdiff_t n; } _GoString_;
    15  #endif
    16  
    17  #endif
    18  
    19  /* Start of preamble from import "C" comments.  */
    20  
    21  
    22  #line 3 "library.go"
    23  
    24  #include "library.h"
    25  
    26  #line 1 "cgo-generated-wrapper"
    27  
    28  
    29  /* End of preamble from import "C" comments.  */
    30  
    31  
    32  /* Start of boilerplate cgo prologue.  */
    33  #line 1 "cgo-gcc-export-header-prolog"
    34  
    35  #ifndef GO_CGO_PROLOGUE_H
    36  #define GO_CGO_PROLOGUE_H
    37  
    38  typedef signed char GoInt8;
    39  typedef unsigned char GoUint8;
    40  typedef short GoInt16;
    41  typedef unsigned short GoUint16;
    42  typedef int GoInt32;
    43  typedef unsigned int GoUint32;
    44  typedef long long GoInt64;
    45  typedef unsigned long long GoUint64;
    46  typedef GoInt64 GoInt;
    47  typedef GoUint64 GoUint;
    48  typedef __SIZE_TYPE__ GoUintptr;
    49  typedef float GoFloat32;
    50  typedef double GoFloat64;
    51  typedef float _Complex GoComplex64;
    52  typedef double _Complex GoComplex128;
    53  
    54  /*
    55    static assertion to make sure the file is being used on architecture
    56    at least with matching size of GoInt.
    57  */
    58  typedef char _check_for_64_bit_pointer_matching_GoInt[sizeof(void*)==64/8 ? 1:-1];
    59  
    60  #ifndef GO_CGO_GOSTRING_TYPEDEF
    61  typedef _GoString_ GoString;
    62  #endif
    63  typedef void *GoMap;
    64  typedef void *GoChan;
    65  typedef struct { void *t; void *v; } GoInterface;
    66  typedef struct { void *data; GoInt len; GoInt cap; } GoSlice;
    67  
    68  #endif
    69  
    70  /* End of boilerplate cgo prologue.  */
    71  
    72  #ifdef __cplusplus
    73  extern "C" {
    74  #endif
    75  
    76  
    77  // initTokenizer accepts a vocabulary id as a C string, and if it does not
    78  // exist in the global tokenizers map, initializes a tokenizer for that
    79  // vocabulary.
    80  extern GoUint8 initTokenizer(char* vocab_id);
    81  extern Tokens tokenizeBuffer(char* vocabIdStr, char* buf, size_t sz);
    82  
    83  // tokenize accepts a vocabulary and text as a C string, and returns a C.Tokens
    84  // that contains a malloc'ed array of uint32_t tokens along with the number of
    85  // tokens.
    86  extern Tokens tokenize(char* vocabIdStr, char* str);
    87  
    88  // decode accepts a vocabulary id and a C.Tokens struct, and returns a malloc'ed
    89  // C.char* containing the decoded string.
    90  extern char* decode(char* vocabIdStr, Tokens tokens);
    91  extern void freeTokens(Tokens tokens);
    92  
    93  #ifdef __cplusplus
    94  }
    95  #endif