github.com/wbrown/gpt_bpe@v0.0.0-20250709161131-1571a6e8ad2d/resources/data/llama3-tokenizer/tokenizer_config.json (about) 1 { 2 "added_tokens_decoder": { 3 "128000": { 4 "content": "<|begin_of_text|>", 5 "lstrip": false, 6 "normalized": false, 7 "rstrip": false, 8 "single_word": false, 9 "special": true 10 }, 11 "128001": { 12 "content": "<|end_of_text|>", 13 "lstrip": false, 14 "normalized": false, 15 "rstrip": false, 16 "single_word": false, 17 "special": true 18 }, 19 "128002": { 20 "content": "<|reserved_special_token_0|>", 21 "lstrip": false, 22 "normalized": false, 23 "rstrip": false, 24 "single_word": false, 25 "special": true 26 }, 27 "128003": { 28 "content": "<|reserved_special_token_1|>", 29 "lstrip": false, 30 "normalized": false, 31 "rstrip": false, 32 "single_word": false, 33 "special": true 34 }, 35 "128004": { 36 "content": "<|reserved_special_token_2|>", 37 "lstrip": false, 38 "normalized": false, 39 "rstrip": false, 40 "single_word": false, 41 "special": true 42 }, 43 "128005": { 44 "content": "<|reserved_special_token_3|>", 45 "lstrip": false, 46 "normalized": false, 47 "rstrip": false, 48 "single_word": false, 49 "special": true 50 }, 51 "128006": { 52 "content": "<|start_header_id|>", 53 "lstrip": false, 54 "normalized": false, 55 "rstrip": false, 56 "single_word": false, 57 "special": true 58 }, 59 "128007": { 60 "content": "<|end_header_id|>", 61 "lstrip": false, 62 "normalized": false, 63 "rstrip": false, 64 "single_word": false, 65 "special": true 66 }, 67 "128008": { 68 "content": "<|reserved_special_token_4|>", 69 "lstrip": false, 70 "normalized": false, 71 "rstrip": false, 72 "single_word": false, 73 "special": true 74 }, 75 "128009": { 76 "content": "<|eot_id|>", 77 "lstrip": false, 78 "normalized": false, 79 "rstrip": false, 80 "single_word": false, 81 "special": true 82 }, 83 "128010": { 84 "content": "<|reserved_special_token_5|>", 85 "lstrip": false, 86 "normalized": false, 87 "rstrip": false, 88 "single_word": false, 89 "special": true 90 }, 91 "128011": { 92 "content": "<|reserved_special_token_6|>", 93 "lstrip": false, 94 "normalized": false, 95 "rstrip": false, 96 "single_word": false, 97 "special": true 98 }, 99 "128012": { 100 "content": "<|reserved_special_token_7|>", 101 "lstrip": false, 102 "normalized": false, 103 "rstrip": false, 104 "single_word": false, 105 "special": true 106 }, 107 "128013": { 108 "content": "<|reserved_special_token_8|>", 109 "lstrip": false, 110 "normalized": false, 111 "rstrip": false, 112 "single_word": false, 113 "special": true 114 }, 115 "128014": { 116 "content": "<|reserved_special_token_9|>", 117 "lstrip": false, 118 "normalized": false, 119 "rstrip": false, 120 "single_word": false, 121 "special": true 122 }, 123 "128015": { 124 "content": "<|reserved_special_token_10|>", 125 "lstrip": false, 126 "normalized": false, 127 "rstrip": false, 128 "single_word": false, 129 "special": true 130 }, 131 "128016": { 132 "content": "<|reserved_special_token_11|>", 133 "lstrip": false, 134 "normalized": false, 135 "rstrip": false, 136 "single_word": false, 137 "special": true 138 }, 139 "128017": { 140 "content": "<|reserved_special_token_12|>", 141 "lstrip": false, 142 "normalized": false, 143 "rstrip": false, 144 "single_word": false, 145 "special": true 146 }, 147 "128018": { 148 "content": "<|reserved_special_token_13|>", 149 "lstrip": false, 150 "normalized": false, 151 "rstrip": false, 152 "single_word": false, 153 "special": true 154 }, 155 "128019": { 156 "content": "<|reserved_special_token_14|>", 157 "lstrip": false, 158 "normalized": false, 159 "rstrip": false, 160 "single_word": false, 161 "special": true 162 }, 163 "128020": { 164 "content": "<|reserved_special_token_15|>", 165 "lstrip": false, 166 "normalized": false, 167 "rstrip": false, 168 "single_word": false, 169 "special": true 170 }, 171 "128021": { 172 "content": "<|reserved_special_token_16|>", 173 "lstrip": false, 174 "normalized": false, 175 "rstrip": false, 176 "single_word": false, 177 "special": true 178 }, 179 "128022": { 180 "content": "<|reserved_special_token_17|>", 181 "lstrip": false, 182 "normalized": false, 183 "rstrip": false, 184 "single_word": false, 185 "special": true 186 }, 187 "128023": { 188 "content": "<|reserved_special_token_18|>", 189 "lstrip": false, 190 "normalized": false, 191 "rstrip": false, 192 "single_word": false, 193 "special": true 194 }, 195 "128024": { 196 "content": "<|reserved_special_token_19|>", 197 "lstrip": false, 198 "normalized": false, 199 "rstrip": false, 200 "single_word": false, 201 "special": true 202 }, 203 "128025": { 204 "content": "<|reserved_special_token_20|>", 205 "lstrip": false, 206 "normalized": false, 207 "rstrip": false, 208 "single_word": false, 209 "special": true 210 }, 211 "128026": { 212 "content": "<|reserved_special_token_21|>", 213 "lstrip": false, 214 "normalized": false, 215 "rstrip": false, 216 "single_word": false, 217 "special": true 218 }, 219 "128027": { 220 "content": "<|reserved_special_token_22|>", 221 "lstrip": false, 222 "normalized": false, 223 "rstrip": false, 224 "single_word": false, 225 "special": true 226 }, 227 "128028": { 228 "content": "<|reserved_special_token_23|>", 229 "lstrip": false, 230 "normalized": false, 231 "rstrip": false, 232 "single_word": false, 233 "special": true 234 }, 235 "128029": { 236 "content": "<|reserved_special_token_24|>", 237 "lstrip": false, 238 "normalized": false, 239 "rstrip": false, 240 "single_word": false, 241 "special": true 242 }, 243 "128030": { 244 "content": "<|reserved_special_token_25|>", 245 "lstrip": false, 246 "normalized": false, 247 "rstrip": false, 248 "single_word": false, 249 "special": true 250 }, 251 "128031": { 252 "content": "<|reserved_special_token_26|>", 253 "lstrip": false, 254 "normalized": false, 255 "rstrip": false, 256 "single_word": false, 257 "special": true 258 }, 259 "128032": { 260 "content": "<|reserved_special_token_27|>", 261 "lstrip": false, 262 "normalized": false, 263 "rstrip": false, 264 "single_word": false, 265 "special": true 266 }, 267 "128033": { 268 "content": "<|reserved_special_token_28|>", 269 "lstrip": false, 270 "normalized": false, 271 "rstrip": false, 272 "single_word": false, 273 "special": true 274 }, 275 "128034": { 276 "content": "<|reserved_special_token_29|>", 277 "lstrip": false, 278 "normalized": false, 279 "rstrip": false, 280 "single_word": false, 281 "special": true 282 }, 283 "128035": { 284 "content": "<|reserved_special_token_30|>", 285 "lstrip": false, 286 "normalized": false, 287 "rstrip": false, 288 "single_word": false, 289 "special": true 290 }, 291 "128036": { 292 "content": "<|reserved_special_token_31|>", 293 "lstrip": false, 294 "normalized": false, 295 "rstrip": false, 296 "single_word": false, 297 "special": true 298 }, 299 "128037": { 300 "content": "<|reserved_special_token_32|>", 301 "lstrip": false, 302 "normalized": false, 303 "rstrip": false, 304 "single_word": false, 305 "special": true 306 }, 307 "128038": { 308 "content": "<|reserved_special_token_33|>", 309 "lstrip": false, 310 "normalized": false, 311 "rstrip": false, 312 "single_word": false, 313 "special": true 314 }, 315 "128039": { 316 "content": "<|reserved_special_token_34|>", 317 "lstrip": false, 318 "normalized": false, 319 "rstrip": false, 320 "single_word": false, 321 "special": true 322 }, 323 "128040": { 324 "content": "<|reserved_special_token_35|>", 325 "lstrip": false, 326 "normalized": false, 327 "rstrip": false, 328 "single_word": false, 329 "special": true 330 }, 331 "128041": { 332 "content": "<|reserved_special_token_36|>", 333 "lstrip": false, 334 "normalized": false, 335 "rstrip": false, 336 "single_word": false, 337 "special": true 338 }, 339 "128042": { 340 "content": "<|reserved_special_token_37|>", 341 "lstrip": false, 342 "normalized": false, 343 "rstrip": false, 344 "single_word": false, 345 "special": true 346 }, 347 "128043": { 348 "content": "<|reserved_special_token_38|>", 349 "lstrip": false, 350 "normalized": false, 351 "rstrip": false, 352 "single_word": false, 353 "special": true 354 }, 355 "128044": { 356 "content": "<|reserved_special_token_39|>", 357 "lstrip": false, 358 "normalized": false, 359 "rstrip": false, 360 "single_word": false, 361 "special": true 362 }, 363 "128045": { 364 "content": "<|reserved_special_token_40|>", 365 "lstrip": false, 366 "normalized": false, 367 "rstrip": false, 368 "single_word": false, 369 "special": true 370 }, 371 "128046": { 372 "content": "<|reserved_special_token_41|>", 373 "lstrip": false, 374 "normalized": false, 375 "rstrip": false, 376 "single_word": false, 377 "special": true 378 }, 379 "128047": { 380 "content": "<|reserved_special_token_42|>", 381 "lstrip": false, 382 "normalized": false, 383 "rstrip": false, 384 "single_word": false, 385 "special": true 386 }, 387 "128048": { 388 "content": "<|reserved_special_token_43|>", 389 "lstrip": false, 390 "normalized": false, 391 "rstrip": false, 392 "single_word": false, 393 "special": true 394 }, 395 "128049": { 396 "content": "<|reserved_special_token_44|>", 397 "lstrip": false, 398 "normalized": false, 399 "rstrip": false, 400 "single_word": false, 401 "special": true 402 }, 403 "128050": { 404 "content": "<|reserved_special_token_45|>", 405 "lstrip": false, 406 "normalized": false, 407 "rstrip": false, 408 "single_word": false, 409 "special": true 410 }, 411 "128051": { 412 "content": "<|reserved_special_token_46|>", 413 "lstrip": false, 414 "normalized": false, 415 "rstrip": false, 416 "single_word": false, 417 "special": true 418 }, 419 "128052": { 420 "content": "<|reserved_special_token_47|>", 421 "lstrip": false, 422 "normalized": false, 423 "rstrip": false, 424 "single_word": false, 425 "special": true 426 }, 427 "128053": { 428 "content": "<|reserved_special_token_48|>", 429 "lstrip": false, 430 "normalized": false, 431 "rstrip": false, 432 "single_word": false, 433 "special": true 434 }, 435 "128054": { 436 "content": "<|reserved_special_token_49|>", 437 "lstrip": false, 438 "normalized": false, 439 "rstrip": false, 440 "single_word": false, 441 "special": true 442 }, 443 "128055": { 444 "content": "<|reserved_special_token_50|>", 445 "lstrip": false, 446 "normalized": false, 447 "rstrip": false, 448 "single_word": false, 449 "special": true 450 }, 451 "128056": { 452 "content": "<|reserved_special_token_51|>", 453 "lstrip": false, 454 "normalized": false, 455 "rstrip": false, 456 "single_word": false, 457 "special": true 458 }, 459 "128057": { 460 "content": "<|reserved_special_token_52|>", 461 "lstrip": false, 462 "normalized": false, 463 "rstrip": false, 464 "single_word": false, 465 "special": true 466 }, 467 "128058": { 468 "content": "<|reserved_special_token_53|>", 469 "lstrip": false, 470 "normalized": false, 471 "rstrip": false, 472 "single_word": false, 473 "special": true 474 }, 475 "128059": { 476 "content": "<|reserved_special_token_54|>", 477 "lstrip": false, 478 "normalized": false, 479 "rstrip": false, 480 "single_word": false, 481 "special": true 482 }, 483 "128060": { 484 "content": "<|reserved_special_token_55|>", 485 "lstrip": false, 486 "normalized": false, 487 "rstrip": false, 488 "single_word": false, 489 "special": true 490 }, 491 "128061": { 492 "content": "<|reserved_special_token_56|>", 493 "lstrip": false, 494 "normalized": false, 495 "rstrip": false, 496 "single_word": false, 497 "special": true 498 }, 499 "128062": { 500 "content": "<|reserved_special_token_57|>", 501 "lstrip": false, 502 "normalized": false, 503 "rstrip": false, 504 "single_word": false, 505 "special": true 506 }, 507 "128063": { 508 "content": "<|reserved_special_token_58|>", 509 "lstrip": false, 510 "normalized": false, 511 "rstrip": false, 512 "single_word": false, 513 "special": true 514 }, 515 "128064": { 516 "content": "<|reserved_special_token_59|>", 517 "lstrip": false, 518 "normalized": false, 519 "rstrip": false, 520 "single_word": false, 521 "special": true 522 }, 523 "128065": { 524 "content": "<|reserved_special_token_60|>", 525 "lstrip": false, 526 "normalized": false, 527 "rstrip": false, 528 "single_word": false, 529 "special": true 530 }, 531 "128066": { 532 "content": "<|reserved_special_token_61|>", 533 "lstrip": false, 534 "normalized": false, 535 "rstrip": false, 536 "single_word": false, 537 "special": true 538 }, 539 "128067": { 540 "content": "<|reserved_special_token_62|>", 541 "lstrip": false, 542 "normalized": false, 543 "rstrip": false, 544 "single_word": false, 545 "special": true 546 }, 547 "128068": { 548 "content": "<|reserved_special_token_63|>", 549 "lstrip": false, 550 "normalized": false, 551 "rstrip": false, 552 "single_word": false, 553 "special": true 554 }, 555 "128069": { 556 "content": "<|reserved_special_token_64|>", 557 "lstrip": false, 558 "normalized": false, 559 "rstrip": false, 560 "single_word": false, 561 "special": true 562 }, 563 "128070": { 564 "content": "<|reserved_special_token_65|>", 565 "lstrip": false, 566 "normalized": false, 567 "rstrip": false, 568 "single_word": false, 569 "special": true 570 }, 571 "128071": { 572 "content": "<|reserved_special_token_66|>", 573 "lstrip": false, 574 "normalized": false, 575 "rstrip": false, 576 "single_word": false, 577 "special": true 578 }, 579 "128072": { 580 "content": "<|reserved_special_token_67|>", 581 "lstrip": false, 582 "normalized": false, 583 "rstrip": false, 584 "single_word": false, 585 "special": true 586 }, 587 "128073": { 588 "content": "<|reserved_special_token_68|>", 589 "lstrip": false, 590 "normalized": false, 591 "rstrip": false, 592 "single_word": false, 593 "special": true 594 }, 595 "128074": { 596 "content": "<|reserved_special_token_69|>", 597 "lstrip": false, 598 "normalized": false, 599 "rstrip": false, 600 "single_word": false, 601 "special": true 602 }, 603 "128075": { 604 "content": "<|reserved_special_token_70|>", 605 "lstrip": false, 606 "normalized": false, 607 "rstrip": false, 608 "single_word": false, 609 "special": true 610 }, 611 "128076": { 612 "content": "<|reserved_special_token_71|>", 613 "lstrip": false, 614 "normalized": false, 615 "rstrip": false, 616 "single_word": false, 617 "special": true 618 }, 619 "128077": { 620 "content": "<|reserved_special_token_72|>", 621 "lstrip": false, 622 "normalized": false, 623 "rstrip": false, 624 "single_word": false, 625 "special": true 626 }, 627 "128078": { 628 "content": "<|reserved_special_token_73|>", 629 "lstrip": false, 630 "normalized": false, 631 "rstrip": false, 632 "single_word": false, 633 "special": true 634 }, 635 "128079": { 636 "content": "<|reserved_special_token_74|>", 637 "lstrip": false, 638 "normalized": false, 639 "rstrip": false, 640 "single_word": false, 641 "special": true 642 }, 643 "128080": { 644 "content": "<|reserved_special_token_75|>", 645 "lstrip": false, 646 "normalized": false, 647 "rstrip": false, 648 "single_word": false, 649 "special": true 650 }, 651 "128081": { 652 "content": "<|reserved_special_token_76|>", 653 "lstrip": false, 654 "normalized": false, 655 "rstrip": false, 656 "single_word": false, 657 "special": true 658 }, 659 "128082": { 660 "content": "<|reserved_special_token_77|>", 661 "lstrip": false, 662 "normalized": false, 663 "rstrip": false, 664 "single_word": false, 665 "special": true 666 }, 667 "128083": { 668 "content": "<|reserved_special_token_78|>", 669 "lstrip": false, 670 "normalized": false, 671 "rstrip": false, 672 "single_word": false, 673 "special": true 674 }, 675 "128084": { 676 "content": "<|reserved_special_token_79|>", 677 "lstrip": false, 678 "normalized": false, 679 "rstrip": false, 680 "single_word": false, 681 "special": true 682 }, 683 "128085": { 684 "content": "<|reserved_special_token_80|>", 685 "lstrip": false, 686 "normalized": false, 687 "rstrip": false, 688 "single_word": false, 689 "special": true 690 }, 691 "128086": { 692 "content": "<|reserved_special_token_81|>", 693 "lstrip": false, 694 "normalized": false, 695 "rstrip": false, 696 "single_word": false, 697 "special": true 698 }, 699 "128087": { 700 "content": "<|reserved_special_token_82|>", 701 "lstrip": false, 702 "normalized": false, 703 "rstrip": false, 704 "single_word": false, 705 "special": true 706 }, 707 "128088": { 708 "content": "<|reserved_special_token_83|>", 709 "lstrip": false, 710 "normalized": false, 711 "rstrip": false, 712 "single_word": false, 713 "special": true 714 }, 715 "128089": { 716 "content": "<|reserved_special_token_84|>", 717 "lstrip": false, 718 "normalized": false, 719 "rstrip": false, 720 "single_word": false, 721 "special": true 722 }, 723 "128090": { 724 "content": "<|reserved_special_token_85|>", 725 "lstrip": false, 726 "normalized": false, 727 "rstrip": false, 728 "single_word": false, 729 "special": true 730 }, 731 "128091": { 732 "content": "<|reserved_special_token_86|>", 733 "lstrip": false, 734 "normalized": false, 735 "rstrip": false, 736 "single_word": false, 737 "special": true 738 }, 739 "128092": { 740 "content": "<|reserved_special_token_87|>", 741 "lstrip": false, 742 "normalized": false, 743 "rstrip": false, 744 "single_word": false, 745 "special": true 746 }, 747 "128093": { 748 "content": "<|reserved_special_token_88|>", 749 "lstrip": false, 750 "normalized": false, 751 "rstrip": false, 752 "single_word": false, 753 "special": true 754 }, 755 "128094": { 756 "content": "<|reserved_special_token_89|>", 757 "lstrip": false, 758 "normalized": false, 759 "rstrip": false, 760 "single_word": false, 761 "special": true 762 }, 763 "128095": { 764 "content": "<|reserved_special_token_90|>", 765 "lstrip": false, 766 "normalized": false, 767 "rstrip": false, 768 "single_word": false, 769 "special": true 770 }, 771 "128096": { 772 "content": "<|reserved_special_token_91|>", 773 "lstrip": false, 774 "normalized": false, 775 "rstrip": false, 776 "single_word": false, 777 "special": true 778 }, 779 "128097": { 780 "content": "<|reserved_special_token_92|>", 781 "lstrip": false, 782 "normalized": false, 783 "rstrip": false, 784 "single_word": false, 785 "special": true 786 }, 787 "128098": { 788 "content": "<|reserved_special_token_93|>", 789 "lstrip": false, 790 "normalized": false, 791 "rstrip": false, 792 "single_word": false, 793 "special": true 794 }, 795 "128099": { 796 "content": "<|reserved_special_token_94|>", 797 "lstrip": false, 798 "normalized": false, 799 "rstrip": false, 800 "single_word": false, 801 "special": true 802 }, 803 "128100": { 804 "content": "<|reserved_special_token_95|>", 805 "lstrip": false, 806 "normalized": false, 807 "rstrip": false, 808 "single_word": false, 809 "special": true 810 }, 811 "128101": { 812 "content": "<|reserved_special_token_96|>", 813 "lstrip": false, 814 "normalized": false, 815 "rstrip": false, 816 "single_word": false, 817 "special": true 818 }, 819 "128102": { 820 "content": "<|reserved_special_token_97|>", 821 "lstrip": false, 822 "normalized": false, 823 "rstrip": false, 824 "single_word": false, 825 "special": true 826 }, 827 "128103": { 828 "content": "<|reserved_special_token_98|>", 829 "lstrip": false, 830 "normalized": false, 831 "rstrip": false, 832 "single_word": false, 833 "special": true 834 }, 835 "128104": { 836 "content": "<|reserved_special_token_99|>", 837 "lstrip": false, 838 "normalized": false, 839 "rstrip": false, 840 "single_word": false, 841 "special": true 842 }, 843 "128105": { 844 "content": "<|reserved_special_token_100|>", 845 "lstrip": false, 846 "normalized": false, 847 "rstrip": false, 848 "single_word": false, 849 "special": true 850 }, 851 "128106": { 852 "content": "<|reserved_special_token_101|>", 853 "lstrip": false, 854 "normalized": false, 855 "rstrip": false, 856 "single_word": false, 857 "special": true 858 }, 859 "128107": { 860 "content": "<|reserved_special_token_102|>", 861 "lstrip": false, 862 "normalized": false, 863 "rstrip": false, 864 "single_word": false, 865 "special": true 866 }, 867 "128108": { 868 "content": "<|reserved_special_token_103|>", 869 "lstrip": false, 870 "normalized": false, 871 "rstrip": false, 872 "single_word": false, 873 "special": true 874 }, 875 "128109": { 876 "content": "<|reserved_special_token_104|>", 877 "lstrip": false, 878 "normalized": false, 879 "rstrip": false, 880 "single_word": false, 881 "special": true 882 }, 883 "128110": { 884 "content": "<|reserved_special_token_105|>", 885 "lstrip": false, 886 "normalized": false, 887 "rstrip": false, 888 "single_word": false, 889 "special": true 890 }, 891 "128111": { 892 "content": "<|reserved_special_token_106|>", 893 "lstrip": false, 894 "normalized": false, 895 "rstrip": false, 896 "single_word": false, 897 "special": true 898 }, 899 "128112": { 900 "content": "<|reserved_special_token_107|>", 901 "lstrip": false, 902 "normalized": false, 903 "rstrip": false, 904 "single_word": false, 905 "special": true 906 }, 907 "128113": { 908 "content": "<|reserved_special_token_108|>", 909 "lstrip": false, 910 "normalized": false, 911 "rstrip": false, 912 "single_word": false, 913 "special": true 914 }, 915 "128114": { 916 "content": "<|reserved_special_token_109|>", 917 "lstrip": false, 918 "normalized": false, 919 "rstrip": false, 920 "single_word": false, 921 "special": true 922 }, 923 "128115": { 924 "content": "<|reserved_special_token_110|>", 925 "lstrip": false, 926 "normalized": false, 927 "rstrip": false, 928 "single_word": false, 929 "special": true 930 }, 931 "128116": { 932 "content": "<|reserved_special_token_111|>", 933 "lstrip": false, 934 "normalized": false, 935 "rstrip": false, 936 "single_word": false, 937 "special": true 938 }, 939 "128117": { 940 "content": "<|reserved_special_token_112|>", 941 "lstrip": false, 942 "normalized": false, 943 "rstrip": false, 944 "single_word": false, 945 "special": true 946 }, 947 "128118": { 948 "content": "<|reserved_special_token_113|>", 949 "lstrip": false, 950 "normalized": false, 951 "rstrip": false, 952 "single_word": false, 953 "special": true 954 }, 955 "128119": { 956 "content": "<|reserved_special_token_114|>", 957 "lstrip": false, 958 "normalized": false, 959 "rstrip": false, 960 "single_word": false, 961 "special": true 962 }, 963 "128120": { 964 "content": "<|reserved_special_token_115|>", 965 "lstrip": false, 966 "normalized": false, 967 "rstrip": false, 968 "single_word": false, 969 "special": true 970 }, 971 "128121": { 972 "content": "<|reserved_special_token_116|>", 973 "lstrip": false, 974 "normalized": false, 975 "rstrip": false, 976 "single_word": false, 977 "special": true 978 }, 979 "128122": { 980 "content": "<|reserved_special_token_117|>", 981 "lstrip": false, 982 "normalized": false, 983 "rstrip": false, 984 "single_word": false, 985 "special": true 986 }, 987 "128123": { 988 "content": "<|reserved_special_token_118|>", 989 "lstrip": false, 990 "normalized": false, 991 "rstrip": false, 992 "single_word": false, 993 "special": true 994 }, 995 "128124": { 996 "content": "<|reserved_special_token_119|>", 997 "lstrip": false, 998 "normalized": false, 999 "rstrip": false, 1000 "single_word": false, 1001 "special": true 1002 }, 1003 "128125": { 1004 "content": "<|reserved_special_token_120|>", 1005 "lstrip": false, 1006 "normalized": false, 1007 "rstrip": false, 1008 "single_word": false, 1009 "special": true 1010 }, 1011 "128126": { 1012 "content": "<|reserved_special_token_121|>", 1013 "lstrip": false, 1014 "normalized": false, 1015 "rstrip": false, 1016 "single_word": false, 1017 "special": true 1018 }, 1019 "128127": { 1020 "content": "<|reserved_special_token_122|>", 1021 "lstrip": false, 1022 "normalized": false, 1023 "rstrip": false, 1024 "single_word": false, 1025 "special": true 1026 }, 1027 "128128": { 1028 "content": "<|reserved_special_token_123|>", 1029 "lstrip": false, 1030 "normalized": false, 1031 "rstrip": false, 1032 "single_word": false, 1033 "special": true 1034 }, 1035 "128129": { 1036 "content": "<|reserved_special_token_124|>", 1037 "lstrip": false, 1038 "normalized": false, 1039 "rstrip": false, 1040 "single_word": false, 1041 "special": true 1042 }, 1043 "128130": { 1044 "content": "<|reserved_special_token_125|>", 1045 "lstrip": false, 1046 "normalized": false, 1047 "rstrip": false, 1048 "single_word": false, 1049 "special": true 1050 }, 1051 "128131": { 1052 "content": "<|reserved_special_token_126|>", 1053 "lstrip": false, 1054 "normalized": false, 1055 "rstrip": false, 1056 "single_word": false, 1057 "special": true 1058 }, 1059 "128132": { 1060 "content": "<|reserved_special_token_127|>", 1061 "lstrip": false, 1062 "normalized": false, 1063 "rstrip": false, 1064 "single_word": false, 1065 "special": true 1066 }, 1067 "128133": { 1068 "content": "<|reserved_special_token_128|>", 1069 "lstrip": false, 1070 "normalized": false, 1071 "rstrip": false, 1072 "single_word": false, 1073 "special": true 1074 }, 1075 "128134": { 1076 "content": "<|reserved_special_token_129|>", 1077 "lstrip": false, 1078 "normalized": false, 1079 "rstrip": false, 1080 "single_word": false, 1081 "special": true 1082 }, 1083 "128135": { 1084 "content": "<|reserved_special_token_130|>", 1085 "lstrip": false, 1086 "normalized": false, 1087 "rstrip": false, 1088 "single_word": false, 1089 "special": true 1090 }, 1091 "128136": { 1092 "content": "<|reserved_special_token_131|>", 1093 "lstrip": false, 1094 "normalized": false, 1095 "rstrip": false, 1096 "single_word": false, 1097 "special": true 1098 }, 1099 "128137": { 1100 "content": "<|reserved_special_token_132|>", 1101 "lstrip": false, 1102 "normalized": false, 1103 "rstrip": false, 1104 "single_word": false, 1105 "special": true 1106 }, 1107 "128138": { 1108 "content": "<|reserved_special_token_133|>", 1109 "lstrip": false, 1110 "normalized": false, 1111 "rstrip": false, 1112 "single_word": false, 1113 "special": true 1114 }, 1115 "128139": { 1116 "content": "<|reserved_special_token_134|>", 1117 "lstrip": false, 1118 "normalized": false, 1119 "rstrip": false, 1120 "single_word": false, 1121 "special": true 1122 }, 1123 "128140": { 1124 "content": "<|reserved_special_token_135|>", 1125 "lstrip": false, 1126 "normalized": false, 1127 "rstrip": false, 1128 "single_word": false, 1129 "special": true 1130 }, 1131 "128141": { 1132 "content": "<|reserved_special_token_136|>", 1133 "lstrip": false, 1134 "normalized": false, 1135 "rstrip": false, 1136 "single_word": false, 1137 "special": true 1138 }, 1139 "128142": { 1140 "content": "<|reserved_special_token_137|>", 1141 "lstrip": false, 1142 "normalized": false, 1143 "rstrip": false, 1144 "single_word": false, 1145 "special": true 1146 }, 1147 "128143": { 1148 "content": "<|reserved_special_token_138|>", 1149 "lstrip": false, 1150 "normalized": false, 1151 "rstrip": false, 1152 "single_word": false, 1153 "special": true 1154 }, 1155 "128144": { 1156 "content": "<|reserved_special_token_139|>", 1157 "lstrip": false, 1158 "normalized": false, 1159 "rstrip": false, 1160 "single_word": false, 1161 "special": true 1162 }, 1163 "128145": { 1164 "content": "<|reserved_special_token_140|>", 1165 "lstrip": false, 1166 "normalized": false, 1167 "rstrip": false, 1168 "single_word": false, 1169 "special": true 1170 }, 1171 "128146": { 1172 "content": "<|reserved_special_token_141|>", 1173 "lstrip": false, 1174 "normalized": false, 1175 "rstrip": false, 1176 "single_word": false, 1177 "special": true 1178 }, 1179 "128147": { 1180 "content": "<|reserved_special_token_142|>", 1181 "lstrip": false, 1182 "normalized": false, 1183 "rstrip": false, 1184 "single_word": false, 1185 "special": true 1186 }, 1187 "128148": { 1188 "content": "<|reserved_special_token_143|>", 1189 "lstrip": false, 1190 "normalized": false, 1191 "rstrip": false, 1192 "single_word": false, 1193 "special": true 1194 }, 1195 "128149": { 1196 "content": "<|reserved_special_token_144|>", 1197 "lstrip": false, 1198 "normalized": false, 1199 "rstrip": false, 1200 "single_word": false, 1201 "special": true 1202 }, 1203 "128150": { 1204 "content": "<|reserved_special_token_145|>", 1205 "lstrip": false, 1206 "normalized": false, 1207 "rstrip": false, 1208 "single_word": false, 1209 "special": true 1210 }, 1211 "128151": { 1212 "content": "<|reserved_special_token_146|>", 1213 "lstrip": false, 1214 "normalized": false, 1215 "rstrip": false, 1216 "single_word": false, 1217 "special": true 1218 }, 1219 "128152": { 1220 "content": "<|reserved_special_token_147|>", 1221 "lstrip": false, 1222 "normalized": false, 1223 "rstrip": false, 1224 "single_word": false, 1225 "special": true 1226 }, 1227 "128153": { 1228 "content": "<|reserved_special_token_148|>", 1229 "lstrip": false, 1230 "normalized": false, 1231 "rstrip": false, 1232 "single_word": false, 1233 "special": true 1234 }, 1235 "128154": { 1236 "content": "<|reserved_special_token_149|>", 1237 "lstrip": false, 1238 "normalized": false, 1239 "rstrip": false, 1240 "single_word": false, 1241 "special": true 1242 }, 1243 "128155": { 1244 "content": "<|reserved_special_token_150|>", 1245 "lstrip": false, 1246 "normalized": false, 1247 "rstrip": false, 1248 "single_word": false, 1249 "special": true 1250 }, 1251 "128156": { 1252 "content": "<|reserved_special_token_151|>", 1253 "lstrip": false, 1254 "normalized": false, 1255 "rstrip": false, 1256 "single_word": false, 1257 "special": true 1258 }, 1259 "128157": { 1260 "content": "<|reserved_special_token_152|>", 1261 "lstrip": false, 1262 "normalized": false, 1263 "rstrip": false, 1264 "single_word": false, 1265 "special": true 1266 }, 1267 "128158": { 1268 "content": "<|reserved_special_token_153|>", 1269 "lstrip": false, 1270 "normalized": false, 1271 "rstrip": false, 1272 "single_word": false, 1273 "special": true 1274 }, 1275 "128159": { 1276 "content": "<|reserved_special_token_154|>", 1277 "lstrip": false, 1278 "normalized": false, 1279 "rstrip": false, 1280 "single_word": false, 1281 "special": true 1282 }, 1283 "128160": { 1284 "content": "<|reserved_special_token_155|>", 1285 "lstrip": false, 1286 "normalized": false, 1287 "rstrip": false, 1288 "single_word": false, 1289 "special": true 1290 }, 1291 "128161": { 1292 "content": "<|reserved_special_token_156|>", 1293 "lstrip": false, 1294 "normalized": false, 1295 "rstrip": false, 1296 "single_word": false, 1297 "special": true 1298 }, 1299 "128162": { 1300 "content": "<|reserved_special_token_157|>", 1301 "lstrip": false, 1302 "normalized": false, 1303 "rstrip": false, 1304 "single_word": false, 1305 "special": true 1306 }, 1307 "128163": { 1308 "content": "<|reserved_special_token_158|>", 1309 "lstrip": false, 1310 "normalized": false, 1311 "rstrip": false, 1312 "single_word": false, 1313 "special": true 1314 }, 1315 "128164": { 1316 "content": "<|reserved_special_token_159|>", 1317 "lstrip": false, 1318 "normalized": false, 1319 "rstrip": false, 1320 "single_word": false, 1321 "special": true 1322 }, 1323 "128165": { 1324 "content": "<|reserved_special_token_160|>", 1325 "lstrip": false, 1326 "normalized": false, 1327 "rstrip": false, 1328 "single_word": false, 1329 "special": true 1330 }, 1331 "128166": { 1332 "content": "<|reserved_special_token_161|>", 1333 "lstrip": false, 1334 "normalized": false, 1335 "rstrip": false, 1336 "single_word": false, 1337 "special": true 1338 }, 1339 "128167": { 1340 "content": "<|reserved_special_token_162|>", 1341 "lstrip": false, 1342 "normalized": false, 1343 "rstrip": false, 1344 "single_word": false, 1345 "special": true 1346 }, 1347 "128168": { 1348 "content": "<|reserved_special_token_163|>", 1349 "lstrip": false, 1350 "normalized": false, 1351 "rstrip": false, 1352 "single_word": false, 1353 "special": true 1354 }, 1355 "128169": { 1356 "content": "<|reserved_special_token_164|>", 1357 "lstrip": false, 1358 "normalized": false, 1359 "rstrip": false, 1360 "single_word": false, 1361 "special": true 1362 }, 1363 "128170": { 1364 "content": "<|reserved_special_token_165|>", 1365 "lstrip": false, 1366 "normalized": false, 1367 "rstrip": false, 1368 "single_word": false, 1369 "special": true 1370 }, 1371 "128171": { 1372 "content": "<|reserved_special_token_166|>", 1373 "lstrip": false, 1374 "normalized": false, 1375 "rstrip": false, 1376 "single_word": false, 1377 "special": true 1378 }, 1379 "128172": { 1380 "content": "<|reserved_special_token_167|>", 1381 "lstrip": false, 1382 "normalized": false, 1383 "rstrip": false, 1384 "single_word": false, 1385 "special": true 1386 }, 1387 "128173": { 1388 "content": "<|reserved_special_token_168|>", 1389 "lstrip": false, 1390 "normalized": false, 1391 "rstrip": false, 1392 "single_word": false, 1393 "special": true 1394 }, 1395 "128174": { 1396 "content": "<|reserved_special_token_169|>", 1397 "lstrip": false, 1398 "normalized": false, 1399 "rstrip": false, 1400 "single_word": false, 1401 "special": true 1402 }, 1403 "128175": { 1404 "content": "<|reserved_special_token_170|>", 1405 "lstrip": false, 1406 "normalized": false, 1407 "rstrip": false, 1408 "single_word": false, 1409 "special": true 1410 }, 1411 "128176": { 1412 "content": "<|reserved_special_token_171|>", 1413 "lstrip": false, 1414 "normalized": false, 1415 "rstrip": false, 1416 "single_word": false, 1417 "special": true 1418 }, 1419 "128177": { 1420 "content": "<|reserved_special_token_172|>", 1421 "lstrip": false, 1422 "normalized": false, 1423 "rstrip": false, 1424 "single_word": false, 1425 "special": true 1426 }, 1427 "128178": { 1428 "content": "<|reserved_special_token_173|>", 1429 "lstrip": false, 1430 "normalized": false, 1431 "rstrip": false, 1432 "single_word": false, 1433 "special": true 1434 }, 1435 "128179": { 1436 "content": "<|reserved_special_token_174|>", 1437 "lstrip": false, 1438 "normalized": false, 1439 "rstrip": false, 1440 "single_word": false, 1441 "special": true 1442 }, 1443 "128180": { 1444 "content": "<|reserved_special_token_175|>", 1445 "lstrip": false, 1446 "normalized": false, 1447 "rstrip": false, 1448 "single_word": false, 1449 "special": true 1450 }, 1451 "128181": { 1452 "content": "<|reserved_special_token_176|>", 1453 "lstrip": false, 1454 "normalized": false, 1455 "rstrip": false, 1456 "single_word": false, 1457 "special": true 1458 }, 1459 "128182": { 1460 "content": "<|reserved_special_token_177|>", 1461 "lstrip": false, 1462 "normalized": false, 1463 "rstrip": false, 1464 "single_word": false, 1465 "special": true 1466 }, 1467 "128183": { 1468 "content": "<|reserved_special_token_178|>", 1469 "lstrip": false, 1470 "normalized": false, 1471 "rstrip": false, 1472 "single_word": false, 1473 "special": true 1474 }, 1475 "128184": { 1476 "content": "<|reserved_special_token_179|>", 1477 "lstrip": false, 1478 "normalized": false, 1479 "rstrip": false, 1480 "single_word": false, 1481 "special": true 1482 }, 1483 "128185": { 1484 "content": "<|reserved_special_token_180|>", 1485 "lstrip": false, 1486 "normalized": false, 1487 "rstrip": false, 1488 "single_word": false, 1489 "special": true 1490 }, 1491 "128186": { 1492 "content": "<|reserved_special_token_181|>", 1493 "lstrip": false, 1494 "normalized": false, 1495 "rstrip": false, 1496 "single_word": false, 1497 "special": true 1498 }, 1499 "128187": { 1500 "content": "<|reserved_special_token_182|>", 1501 "lstrip": false, 1502 "normalized": false, 1503 "rstrip": false, 1504 "single_word": false, 1505 "special": true 1506 }, 1507 "128188": { 1508 "content": "<|reserved_special_token_183|>", 1509 "lstrip": false, 1510 "normalized": false, 1511 "rstrip": false, 1512 "single_word": false, 1513 "special": true 1514 }, 1515 "128189": { 1516 "content": "<|reserved_special_token_184|>", 1517 "lstrip": false, 1518 "normalized": false, 1519 "rstrip": false, 1520 "single_word": false, 1521 "special": true 1522 }, 1523 "128190": { 1524 "content": "<|reserved_special_token_185|>", 1525 "lstrip": false, 1526 "normalized": false, 1527 "rstrip": false, 1528 "single_word": false, 1529 "special": true 1530 }, 1531 "128191": { 1532 "content": "<|reserved_special_token_186|>", 1533 "lstrip": false, 1534 "normalized": false, 1535 "rstrip": false, 1536 "single_word": false, 1537 "special": true 1538 }, 1539 "128192": { 1540 "content": "<|reserved_special_token_187|>", 1541 "lstrip": false, 1542 "normalized": false, 1543 "rstrip": false, 1544 "single_word": false, 1545 "special": true 1546 }, 1547 "128193": { 1548 "content": "<|reserved_special_token_188|>", 1549 "lstrip": false, 1550 "normalized": false, 1551 "rstrip": false, 1552 "single_word": false, 1553 "special": true 1554 }, 1555 "128194": { 1556 "content": "<|reserved_special_token_189|>", 1557 "lstrip": false, 1558 "normalized": false, 1559 "rstrip": false, 1560 "single_word": false, 1561 "special": true 1562 }, 1563 "128195": { 1564 "content": "<|reserved_special_token_190|>", 1565 "lstrip": false, 1566 "normalized": false, 1567 "rstrip": false, 1568 "single_word": false, 1569 "special": true 1570 }, 1571 "128196": { 1572 "content": "<|reserved_special_token_191|>", 1573 "lstrip": false, 1574 "normalized": false, 1575 "rstrip": false, 1576 "single_word": false, 1577 "special": true 1578 }, 1579 "128197": { 1580 "content": "<|reserved_special_token_192|>", 1581 "lstrip": false, 1582 "normalized": false, 1583 "rstrip": false, 1584 "single_word": false, 1585 "special": true 1586 }, 1587 "128198": { 1588 "content": "<|reserved_special_token_193|>", 1589 "lstrip": false, 1590 "normalized": false, 1591 "rstrip": false, 1592 "single_word": false, 1593 "special": true 1594 }, 1595 "128199": { 1596 "content": "<|reserved_special_token_194|>", 1597 "lstrip": false, 1598 "normalized": false, 1599 "rstrip": false, 1600 "single_word": false, 1601 "special": true 1602 }, 1603 "128200": { 1604 "content": "<|reserved_special_token_195|>", 1605 "lstrip": false, 1606 "normalized": false, 1607 "rstrip": false, 1608 "single_word": false, 1609 "special": true 1610 }, 1611 "128201": { 1612 "content": "<|reserved_special_token_196|>", 1613 "lstrip": false, 1614 "normalized": false, 1615 "rstrip": false, 1616 "single_word": false, 1617 "special": true 1618 }, 1619 "128202": { 1620 "content": "<|reserved_special_token_197|>", 1621 "lstrip": false, 1622 "normalized": false, 1623 "rstrip": false, 1624 "single_word": false, 1625 "special": true 1626 }, 1627 "128203": { 1628 "content": "<|reserved_special_token_198|>", 1629 "lstrip": false, 1630 "normalized": false, 1631 "rstrip": false, 1632 "single_word": false, 1633 "special": true 1634 }, 1635 "128204": { 1636 "content": "<|reserved_special_token_199|>", 1637 "lstrip": false, 1638 "normalized": false, 1639 "rstrip": false, 1640 "single_word": false, 1641 "special": true 1642 }, 1643 "128205": { 1644 "content": "<|reserved_special_token_200|>", 1645 "lstrip": false, 1646 "normalized": false, 1647 "rstrip": false, 1648 "single_word": false, 1649 "special": true 1650 }, 1651 "128206": { 1652 "content": "<|reserved_special_token_201|>", 1653 "lstrip": false, 1654 "normalized": false, 1655 "rstrip": false, 1656 "single_word": false, 1657 "special": true 1658 }, 1659 "128207": { 1660 "content": "<|reserved_special_token_202|>", 1661 "lstrip": false, 1662 "normalized": false, 1663 "rstrip": false, 1664 "single_word": false, 1665 "special": true 1666 }, 1667 "128208": { 1668 "content": "<|reserved_special_token_203|>", 1669 "lstrip": false, 1670 "normalized": false, 1671 "rstrip": false, 1672 "single_word": false, 1673 "special": true 1674 }, 1675 "128209": { 1676 "content": "<|reserved_special_token_204|>", 1677 "lstrip": false, 1678 "normalized": false, 1679 "rstrip": false, 1680 "single_word": false, 1681 "special": true 1682 }, 1683 "128210": { 1684 "content": "<|reserved_special_token_205|>", 1685 "lstrip": false, 1686 "normalized": false, 1687 "rstrip": false, 1688 "single_word": false, 1689 "special": true 1690 }, 1691 "128211": { 1692 "content": "<|reserved_special_token_206|>", 1693 "lstrip": false, 1694 "normalized": false, 1695 "rstrip": false, 1696 "single_word": false, 1697 "special": true 1698 }, 1699 "128212": { 1700 "content": "<|reserved_special_token_207|>", 1701 "lstrip": false, 1702 "normalized": false, 1703 "rstrip": false, 1704 "single_word": false, 1705 "special": true 1706 }, 1707 "128213": { 1708 "content": "<|reserved_special_token_208|>", 1709 "lstrip": false, 1710 "normalized": false, 1711 "rstrip": false, 1712 "single_word": false, 1713 "special": true 1714 }, 1715 "128214": { 1716 "content": "<|reserved_special_token_209|>", 1717 "lstrip": false, 1718 "normalized": false, 1719 "rstrip": false, 1720 "single_word": false, 1721 "special": true 1722 }, 1723 "128215": { 1724 "content": "<|reserved_special_token_210|>", 1725 "lstrip": false, 1726 "normalized": false, 1727 "rstrip": false, 1728 "single_word": false, 1729 "special": true 1730 }, 1731 "128216": { 1732 "content": "<|reserved_special_token_211|>", 1733 "lstrip": false, 1734 "normalized": false, 1735 "rstrip": false, 1736 "single_word": false, 1737 "special": true 1738 }, 1739 "128217": { 1740 "content": "<|reserved_special_token_212|>", 1741 "lstrip": false, 1742 "normalized": false, 1743 "rstrip": false, 1744 "single_word": false, 1745 "special": true 1746 }, 1747 "128218": { 1748 "content": "<|reserved_special_token_213|>", 1749 "lstrip": false, 1750 "normalized": false, 1751 "rstrip": false, 1752 "single_word": false, 1753 "special": true 1754 }, 1755 "128219": { 1756 "content": "<|reserved_special_token_214|>", 1757 "lstrip": false, 1758 "normalized": false, 1759 "rstrip": false, 1760 "single_word": false, 1761 "special": true 1762 }, 1763 "128220": { 1764 "content": "<|reserved_special_token_215|>", 1765 "lstrip": false, 1766 "normalized": false, 1767 "rstrip": false, 1768 "single_word": false, 1769 "special": true 1770 }, 1771 "128221": { 1772 "content": "<|reserved_special_token_216|>", 1773 "lstrip": false, 1774 "normalized": false, 1775 "rstrip": false, 1776 "single_word": false, 1777 "special": true 1778 }, 1779 "128222": { 1780 "content": "<|reserved_special_token_217|>", 1781 "lstrip": false, 1782 "normalized": false, 1783 "rstrip": false, 1784 "single_word": false, 1785 "special": true 1786 }, 1787 "128223": { 1788 "content": "<|reserved_special_token_218|>", 1789 "lstrip": false, 1790 "normalized": false, 1791 "rstrip": false, 1792 "single_word": false, 1793 "special": true 1794 }, 1795 "128224": { 1796 "content": "<|reserved_special_token_219|>", 1797 "lstrip": false, 1798 "normalized": false, 1799 "rstrip": false, 1800 "single_word": false, 1801 "special": true 1802 }, 1803 "128225": { 1804 "content": "<|reserved_special_token_220|>", 1805 "lstrip": false, 1806 "normalized": false, 1807 "rstrip": false, 1808 "single_word": false, 1809 "special": true 1810 }, 1811 "128226": { 1812 "content": "<|reserved_special_token_221|>", 1813 "lstrip": false, 1814 "normalized": false, 1815 "rstrip": false, 1816 "single_word": false, 1817 "special": true 1818 }, 1819 "128227": { 1820 "content": "<|reserved_special_token_222|>", 1821 "lstrip": false, 1822 "normalized": false, 1823 "rstrip": false, 1824 "single_word": false, 1825 "special": true 1826 }, 1827 "128228": { 1828 "content": "<|reserved_special_token_223|>", 1829 "lstrip": false, 1830 "normalized": false, 1831 "rstrip": false, 1832 "single_word": false, 1833 "special": true 1834 }, 1835 "128229": { 1836 "content": "<|reserved_special_token_224|>", 1837 "lstrip": false, 1838 "normalized": false, 1839 "rstrip": false, 1840 "single_word": false, 1841 "special": true 1842 }, 1843 "128230": { 1844 "content": "<|reserved_special_token_225|>", 1845 "lstrip": false, 1846 "normalized": false, 1847 "rstrip": false, 1848 "single_word": false, 1849 "special": true 1850 }, 1851 "128231": { 1852 "content": "<|reserved_special_token_226|>", 1853 "lstrip": false, 1854 "normalized": false, 1855 "rstrip": false, 1856 "single_word": false, 1857 "special": true 1858 }, 1859 "128232": { 1860 "content": "<|reserved_special_token_227|>", 1861 "lstrip": false, 1862 "normalized": false, 1863 "rstrip": false, 1864 "single_word": false, 1865 "special": true 1866 }, 1867 "128233": { 1868 "content": "<|reserved_special_token_228|>", 1869 "lstrip": false, 1870 "normalized": false, 1871 "rstrip": false, 1872 "single_word": false, 1873 "special": true 1874 }, 1875 "128234": { 1876 "content": "<|reserved_special_token_229|>", 1877 "lstrip": false, 1878 "normalized": false, 1879 "rstrip": false, 1880 "single_word": false, 1881 "special": true 1882 }, 1883 "128235": { 1884 "content": "<|reserved_special_token_230|>", 1885 "lstrip": false, 1886 "normalized": false, 1887 "rstrip": false, 1888 "single_word": false, 1889 "special": true 1890 }, 1891 "128236": { 1892 "content": "<|reserved_special_token_231|>", 1893 "lstrip": false, 1894 "normalized": false, 1895 "rstrip": false, 1896 "single_word": false, 1897 "special": true 1898 }, 1899 "128237": { 1900 "content": "<|reserved_special_token_232|>", 1901 "lstrip": false, 1902 "normalized": false, 1903 "rstrip": false, 1904 "single_word": false, 1905 "special": true 1906 }, 1907 "128238": { 1908 "content": "<|reserved_special_token_233|>", 1909 "lstrip": false, 1910 "normalized": false, 1911 "rstrip": false, 1912 "single_word": false, 1913 "special": true 1914 }, 1915 "128239": { 1916 "content": "<|reserved_special_token_234|>", 1917 "lstrip": false, 1918 "normalized": false, 1919 "rstrip": false, 1920 "single_word": false, 1921 "special": true 1922 }, 1923 "128240": { 1924 "content": "<|reserved_special_token_235|>", 1925 "lstrip": false, 1926 "normalized": false, 1927 "rstrip": false, 1928 "single_word": false, 1929 "special": true 1930 }, 1931 "128241": { 1932 "content": "<|reserved_special_token_236|>", 1933 "lstrip": false, 1934 "normalized": false, 1935 "rstrip": false, 1936 "single_word": false, 1937 "special": true 1938 }, 1939 "128242": { 1940 "content": "<|reserved_special_token_237|>", 1941 "lstrip": false, 1942 "normalized": false, 1943 "rstrip": false, 1944 "single_word": false, 1945 "special": true 1946 }, 1947 "128243": { 1948 "content": "<|reserved_special_token_238|>", 1949 "lstrip": false, 1950 "normalized": false, 1951 "rstrip": false, 1952 "single_word": false, 1953 "special": true 1954 }, 1955 "128244": { 1956 "content": "<|reserved_special_token_239|>", 1957 "lstrip": false, 1958 "normalized": false, 1959 "rstrip": false, 1960 "single_word": false, 1961 "special": true 1962 }, 1963 "128245": { 1964 "content": "<|reserved_special_token_240|>", 1965 "lstrip": false, 1966 "normalized": false, 1967 "rstrip": false, 1968 "single_word": false, 1969 "special": true 1970 }, 1971 "128246": { 1972 "content": "<|reserved_special_token_241|>", 1973 "lstrip": false, 1974 "normalized": false, 1975 "rstrip": false, 1976 "single_word": false, 1977 "special": true 1978 }, 1979 "128247": { 1980 "content": "<|reserved_special_token_242|>", 1981 "lstrip": false, 1982 "normalized": false, 1983 "rstrip": false, 1984 "single_word": false, 1985 "special": true 1986 }, 1987 "128248": { 1988 "content": "<|reserved_special_token_243|>", 1989 "lstrip": false, 1990 "normalized": false, 1991 "rstrip": false, 1992 "single_word": false, 1993 "special": true 1994 }, 1995 "128249": { 1996 "content": "<|reserved_special_token_244|>", 1997 "lstrip": false, 1998 "normalized": false, 1999 "rstrip": false, 2000 "single_word": false, 2001 "special": true 2002 }, 2003 "128250": { 2004 "content": "<|reserved_special_token_245|>", 2005 "lstrip": false, 2006 "normalized": false, 2007 "rstrip": false, 2008 "single_word": false, 2009 "special": true 2010 }, 2011 "128251": { 2012 "content": "<|reserved_special_token_246|>", 2013 "lstrip": false, 2014 "normalized": false, 2015 "rstrip": false, 2016 "single_word": false, 2017 "special": true 2018 }, 2019 "128252": { 2020 "content": "<|reserved_special_token_247|>", 2021 "lstrip": false, 2022 "normalized": false, 2023 "rstrip": false, 2024 "single_word": false, 2025 "special": true 2026 }, 2027 "128253": { 2028 "content": "<|reserved_special_token_248|>", 2029 "lstrip": false, 2030 "normalized": false, 2031 "rstrip": false, 2032 "single_word": false, 2033 "special": true 2034 }, 2035 "128254": { 2036 "content": "<|reserved_special_token_249|>", 2037 "lstrip": false, 2038 "normalized": false, 2039 "rstrip": false, 2040 "single_word": false, 2041 "special": true 2042 }, 2043 "128255": { 2044 "content": "<|reserved_special_token_250|>", 2045 "lstrip": false, 2046 "normalized": false, 2047 "rstrip": false, 2048 "single_word": false, 2049 "special": true 2050 } 2051 }, 2052 "bos_token": "<|begin_of_text|>", 2053 "clean_up_tokenization_spaces": true, 2054 "eos_token": "<|end_of_text|>", 2055 "model_input_names": [ 2056 "input_ids", 2057 "attention_mask" 2058 ], 2059 "model_max_length": 1000000000000000019884624838656, 2060 "tokenizer_class": "PreTrainedTokenizerFast" 2061 }