github.com/wbrown/gpt_bpe@v0.0.0-20250709161131-1571a6e8ad2d/js/js.go (about)

     1  package main
     2  
     3  //go:generate gopherjs build --minify
     4  
     5  import (
     6  	"log"
     7  
     8  	"github.com/gopherjs/gopherjs/js"
     9  	"github.com/wbrown/gpt_bpe"
    10  	"github.com/wbrown/gpt_bpe/types"
    11  )
    12  
    13  var encoder gpt_bpe.GPTEncoder
    14  
    15  func Tokenize(text string) gpt_bpe.Tokens {
    16  	return *encoder.Encode(&text)
    17  }
    18  
    19  func Decode(arr []byte) string {
    20  	tokens := types.TokensFromBin(&arr)
    21  	return encoder.Decode(tokens)
    22  }
    23  
    24  func init() {
    25  	encoder = gpt_bpe.NewGPT2Encoder()
    26  	js.Module.Get("exports").Set("decode", Decode)
    27  	js.Module.Get("exports").Set("tokenize", Tokenize)
    28  	log.Printf("GPT-2 BPE Decoder Loaded")
    29  }
    30  
    31  func main() {
    32  
    33  }