github.com/wbrown/gpt_bpe@v0.0.0-20250709161131-1571a6e8ad2d/resources/data/mistral-tokenizer/tokenizer_config.json (about)

     1  {
     2    "add_bos_token": true,
     3    "add_eos_token": false,
     4    "added_tokens_decoder": {
     5      "0": {
     6        "content": "<unk>",
     7        "lstrip": false,
     8        "normalized": false,
     9        "rstrip": false,
    10        "single_word": false,
    11        "special": true
    12      },
    13      "1": {
    14        "content": "<s>",
    15        "lstrip": false,
    16        "normalized": false,
    17        "rstrip": false,
    18        "single_word": false,
    19        "special": true
    20      },
    21      "2": {
    22        "content": "</s>",
    23        "lstrip": false,
    24        "normalized": false,
    25        "rstrip": false,
    26        "single_word": false,
    27        "special": true
    28      }
    29    },
    30    "additional_special_tokens": [],
    31    "bos_token": "<s>",
    32    "clean_up_tokenization_spaces": false,
    33    "eos_token": "</s>",
    34    "legacy": true,
    35    "model_max_length": 1000000000000000019884624838656,
    36    "pad_token": null,
    37    "sp_model_kwargs": {},
    38    "spaces_between_special_tokens": false,
    39    "tokenizer_class": "LlamaTokenizer",
    40    "unk_token": "<unk>",
    41    "use_default_system_prompt": false
    42  }