github.com/wbrown/gpt_bpe@v0.0.0-20250709161131-1571a6e8ad2d/resources/data/clip-tokenizer/special_tokens_map.json (about)

     1  {
     2    "bos_token": {
     3      "content": "<|startoftext|>",
     4      "single_word": false,
     5      "lstrip": false,
     6      "rstrip": false,
     7      "normalized": true
     8    },
     9    "eos_token": {
    10      "content": "<|endoftext|>",
    11      "single_word": false,
    12      "lstrip": false,
    13      "rstrip": false,
    14      "normalized": true
    15    },
    16    "unk_token": {
    17      "content": "<|endoftext|>",
    18      "single_word": false,
    19      "lstrip": false,
    20      "rstrip": false,
    21      "normalized": true
    22    },
    23    "pad_token": "<|endoftext|>"
    24  }