arabic_Qwen_chat_llamafile / tokenizer_config.json
HeshamHaroon's picture
Upload tokenizer
9b8fb59 verified
{
"add_bos_token": false,
"add_prefix_space": false,
"added_tokens_decoder": {
"151643": {
"content": "<|endoftext|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151644": {
"content": "<|im_start|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151645": {
"content": "<|im_end|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151646": {
"content": "<|extra_0|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151647": {
"content": "<|extra_1|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151648": {
"content": "<|extra_2|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151649": {
"content": "<|extra_3|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151650": {
"content": "<|extra_4|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151651": {
"content": "<|extra_5|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151652": {
"content": "<|extra_6|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151653": {
"content": "<|extra_7|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151654": {
"content": "<|extra_8|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151655": {
"content": "<|extra_9|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151656": {
"content": "<|extra_10|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151657": {
"content": "<|extra_11|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151658": {
"content": "<|extra_12|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151659": {
"content": "<|extra_13|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151660": {
"content": "<|extra_14|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151661": {
"content": "<|extra_15|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151662": {
"content": "<|extra_16|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151663": {
"content": "<|extra_17|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151664": {
"content": "<|extra_18|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151665": {
"content": "<|extra_19|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151666": {
"content": "<|extra_20|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151667": {
"content": "<|extra_21|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151668": {
"content": "<|extra_22|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151669": {
"content": "<|extra_23|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151670": {
"content": "<|extra_24|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151671": {
"content": "<|extra_25|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151672": {
"content": "<|extra_26|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151673": {
"content": "<|extra_27|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151674": {
"content": "<|extra_28|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151675": {
"content": "<|extra_29|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151676": {
"content": "<|extra_30|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151677": {
"content": "<|extra_31|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151678": {
"content": "<|extra_32|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151679": {
"content": "<|extra_33|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151680": {
"content": "<|extra_34|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151681": {
"content": "<|extra_35|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151682": {
"content": "<|extra_36|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151683": {
"content": "<|extra_37|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151684": {
"content": "<|extra_38|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151685": {
"content": "<|extra_39|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151686": {
"content": "<|extra_40|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151687": {
"content": "<|extra_41|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151688": {
"content": "<|extra_42|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151689": {
"content": "<|extra_43|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151690": {
"content": "<|extra_44|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151691": {
"content": "<|extra_45|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151692": {
"content": "<|extra_46|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151693": {
"content": "<|extra_47|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151694": {
"content": "<|extra_48|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151695": {
"content": "<|extra_49|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151696": {
"content": "<|extra_50|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151697": {
"content": "<|extra_51|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151698": {
"content": "<|extra_52|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151699": {
"content": "<|extra_53|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151700": {
"content": "<|extra_54|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151701": {
"content": "<|extra_55|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151702": {
"content": "<|extra_56|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151703": {
"content": "<|extra_57|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151704": {
"content": "<|extra_58|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151705": {
"content": "<|extra_59|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151706": {
"content": "<|extra_60|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151707": {
"content": "<|extra_61|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151708": {
"content": "<|extra_62|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151709": {
"content": "<|extra_63|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151710": {
"content": "<|extra_64|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151711": {
"content": "<|extra_65|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151712": {
"content": "<|extra_66|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151713": {
"content": "<|extra_67|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151714": {
"content": "<|extra_68|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151715": {
"content": "<|extra_69|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151716": {
"content": "<|extra_70|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151717": {
"content": "<|extra_71|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151718": {
"content": "<|extra_72|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151719": {
"content": "<|extra_73|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151720": {
"content": "<|extra_74|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151721": {
"content": "<|extra_75|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151722": {
"content": "<|extra_76|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151723": {
"content": "<|extra_77|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151724": {
"content": "<|extra_78|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151725": {
"content": "<|extra_79|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151726": {
"content": "<|extra_80|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151727": {
"content": "<|extra_81|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151728": {
"content": "<|extra_82|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151729": {
"content": "<|extra_83|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151730": {
"content": "<|extra_84|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151731": {
"content": "<|extra_85|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151732": {
"content": "<|extra_86|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151733": {
"content": "<|extra_87|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151734": {
"content": "<|extra_88|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151735": {
"content": "<|extra_89|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151736": {
"content": "<|extra_90|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151737": {
"content": "<|extra_91|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151738": {
"content": "<|extra_92|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151739": {
"content": "<|extra_93|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151740": {
"content": "<|extra_94|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151741": {
"content": "<|extra_95|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151742": {
"content": "<|extra_96|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151743": {
"content": "<|extra_97|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151744": {
"content": "<|extra_98|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151745": {
"content": "<|extra_99|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151746": {
"content": "<|extra_100|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151747": {
"content": "<|extra_101|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151748": {
"content": "<|extra_102|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151749": {
"content": "<|extra_103|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151750": {
"content": "<|extra_104|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151751": {
"content": "<|extra_105|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151752": {
"content": "<|extra_106|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151753": {
"content": "<|extra_107|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151754": {
"content": "<|extra_108|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151755": {
"content": "<|extra_109|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151756": {
"content": "<|extra_110|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151757": {
"content": "<|extra_111|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151758": {
"content": "<|extra_112|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151759": {
"content": "<|extra_113|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151760": {
"content": "<|extra_114|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151761": {
"content": "<|extra_115|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151762": {
"content": "<|extra_116|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151763": {
"content": "<|extra_117|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151764": {
"content": "<|extra_118|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151765": {
"content": "<|extra_119|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151766": {
"content": "<|extra_120|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151767": {
"content": "<|extra_121|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151768": {
"content": "<|extra_122|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151769": {
"content": "<|extra_123|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151770": {
"content": "<|extra_124|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151771": {
"content": "<|extra_125|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151772": {
"content": "<|extra_126|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151773": {
"content": "<|extra_127|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151774": {
"content": "<|extra_128|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151775": {
"content": "<|extra_129|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151776": {
"content": "<|extra_130|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151777": {
"content": "<|extra_131|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151778": {
"content": "<|extra_132|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151779": {
"content": "<|extra_133|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151780": {
"content": "<|extra_134|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151781": {
"content": "<|extra_135|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151782": {
"content": "<|extra_136|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151783": {
"content": "<|extra_137|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151784": {
"content": "<|extra_138|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151785": {
"content": "<|extra_139|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151786": {
"content": "<|extra_140|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151787": {
"content": "<|extra_141|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151788": {
"content": "<|extra_142|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151789": {
"content": "<|extra_143|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151790": {
"content": "<|extra_144|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151791": {
"content": "<|extra_145|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151792": {
"content": "<|extra_146|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151793": {
"content": "<|extra_147|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151794": {
"content": "<|extra_148|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151795": {
"content": "<|extra_149|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151796": {
"content": "<|extra_150|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151797": {
"content": "<|extra_151|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151798": {
"content": "<|extra_152|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151799": {
"content": "<|extra_153|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151800": {
"content": "<|extra_154|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151801": {
"content": "<|extra_155|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151802": {
"content": "<|extra_156|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151803": {
"content": "<|extra_157|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151804": {
"content": "<|extra_158|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151805": {
"content": "<|extra_159|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151806": {
"content": "<|extra_160|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151807": {
"content": "<|extra_161|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151808": {
"content": "<|extra_162|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151809": {
"content": "<|extra_163|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151810": {
"content": "<|extra_164|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151811": {
"content": "<|extra_165|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151812": {
"content": "<|extra_166|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151813": {
"content": "<|extra_167|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151814": {
"content": "<|extra_168|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151815": {
"content": "<|extra_169|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151816": {
"content": "<|extra_170|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151817": {
"content": "<|extra_171|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151818": {
"content": "<|extra_172|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151819": {
"content": "<|extra_173|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151820": {
"content": "<|extra_174|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151821": {
"content": "<|extra_175|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151822": {
"content": "<|extra_176|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151823": {
"content": "<|extra_177|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151824": {
"content": "<|extra_178|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151825": {
"content": "<|extra_179|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151826": {
"content": "<|extra_180|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151827": {
"content": "<|extra_181|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151828": {
"content": "<|extra_182|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151829": {
"content": "<|extra_183|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151830": {
"content": "<|extra_184|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151831": {
"content": "<|extra_185|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151832": {
"content": "<|extra_186|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151833": {
"content": "<|extra_187|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151834": {
"content": "<|extra_188|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151835": {
"content": "<|extra_189|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151836": {
"content": "<|extra_190|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151837": {
"content": "<|extra_191|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151838": {
"content": "<|extra_192|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151839": {
"content": "<|extra_193|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151840": {
"content": "<|extra_194|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151841": {
"content": "<|extra_195|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151842": {
"content": "<|extra_196|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151843": {
"content": "<|extra_197|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151844": {
"content": "<|extra_198|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151845": {
"content": "<|extra_199|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151846": {
"content": "<|extra_200|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151847": {
"content": "<|extra_201|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151848": {
"content": "<|extra_202|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151849": {
"content": "<|extra_203|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151850": {
"content": "<|extra_204|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
}
},
"additional_special_tokens": [
"<|im_start|>",
"<|im_end|>",
"<|extra_0|>",
"<|extra_1|>",
"<|extra_2|>",
"<|extra_3|>",
"<|extra_4|>",
"<|extra_5|>",
"<|extra_6|>",
"<|extra_7|>",
"<|extra_8|>",
"<|extra_9|>",
"<|extra_10|>",
"<|extra_11|>",
"<|extra_12|>",
"<|extra_13|>",
"<|extra_14|>",
"<|extra_15|>",
"<|extra_16|>",
"<|extra_17|>",
"<|extra_18|>",
"<|extra_19|>",
"<|extra_20|>",
"<|extra_21|>",
"<|extra_22|>",
"<|extra_23|>",
"<|extra_24|>",
"<|extra_25|>",
"<|extra_26|>",
"<|extra_27|>",
"<|extra_28|>",
"<|extra_29|>",
"<|extra_30|>",
"<|extra_31|>",
"<|extra_32|>",
"<|extra_33|>",
"<|extra_34|>",
"<|extra_35|>",
"<|extra_36|>",
"<|extra_37|>",
"<|extra_38|>",
"<|extra_39|>",
"<|extra_40|>",
"<|extra_41|>",
"<|extra_42|>",
"<|extra_43|>",
"<|extra_44|>",
"<|extra_45|>",
"<|extra_46|>",
"<|extra_47|>",
"<|extra_48|>",
"<|extra_49|>",
"<|extra_50|>",
"<|extra_51|>",
"<|extra_52|>",
"<|extra_53|>",
"<|extra_54|>",
"<|extra_55|>",
"<|extra_56|>",
"<|extra_57|>",
"<|extra_58|>",
"<|extra_59|>",
"<|extra_60|>",
"<|extra_61|>",
"<|extra_62|>",
"<|extra_63|>",
"<|extra_64|>",
"<|extra_65|>",
"<|extra_66|>",
"<|extra_67|>",
"<|extra_68|>",
"<|extra_69|>",
"<|extra_70|>",
"<|extra_71|>",
"<|extra_72|>",
"<|extra_73|>",
"<|extra_74|>",
"<|extra_75|>",
"<|extra_76|>",
"<|extra_77|>",
"<|extra_78|>",
"<|extra_79|>",
"<|extra_80|>",
"<|extra_81|>",
"<|extra_82|>",
"<|extra_83|>",
"<|extra_84|>",
"<|extra_85|>",
"<|extra_86|>",
"<|extra_87|>",
"<|extra_88|>",
"<|extra_89|>",
"<|extra_90|>",
"<|extra_91|>",
"<|extra_92|>",
"<|extra_93|>",
"<|extra_94|>",
"<|extra_95|>",
"<|extra_96|>",
"<|extra_97|>",
"<|extra_98|>",
"<|extra_99|>",
"<|extra_100|>",
"<|extra_101|>",
"<|extra_102|>",
"<|extra_103|>",
"<|extra_104|>",
"<|extra_105|>",
"<|extra_106|>",
"<|extra_107|>",
"<|extra_108|>",
"<|extra_109|>",
"<|extra_110|>",
"<|extra_111|>",
"<|extra_112|>",
"<|extra_113|>",
"<|extra_114|>",
"<|extra_115|>",
"<|extra_116|>",
"<|extra_117|>",
"<|extra_118|>",
"<|extra_119|>",
"<|extra_120|>",
"<|extra_121|>",
"<|extra_122|>",
"<|extra_123|>",
"<|extra_124|>",
"<|extra_125|>",
"<|extra_126|>",
"<|extra_127|>",
"<|extra_128|>",
"<|extra_129|>",
"<|extra_130|>",
"<|extra_131|>",
"<|extra_132|>",
"<|extra_133|>",
"<|extra_134|>",
"<|extra_135|>",
"<|extra_136|>",
"<|extra_137|>",
"<|extra_138|>",
"<|extra_139|>",
"<|extra_140|>",
"<|extra_141|>",
"<|extra_142|>",
"<|extra_143|>",
"<|extra_144|>",
"<|extra_145|>",
"<|extra_146|>",
"<|extra_147|>",
"<|extra_148|>",
"<|extra_149|>",
"<|extra_150|>",
"<|extra_151|>",
"<|extra_152|>",
"<|extra_153|>",
"<|extra_154|>",
"<|extra_155|>",
"<|extra_156|>",
"<|extra_157|>",
"<|extra_158|>",
"<|extra_159|>",
"<|extra_160|>",
"<|extra_161|>",
"<|extra_162|>",
"<|extra_163|>",
"<|extra_164|>",
"<|extra_165|>",
"<|extra_166|>",
"<|extra_167|>",
"<|extra_168|>",
"<|extra_169|>",
"<|extra_170|>",
"<|extra_171|>",
"<|extra_172|>",
"<|extra_173|>",
"<|extra_174|>",
"<|extra_175|>",
"<|extra_176|>",
"<|extra_177|>",
"<|extra_178|>",
"<|extra_179|>",
"<|extra_180|>",
"<|extra_181|>",
"<|extra_182|>",
"<|extra_183|>",
"<|extra_184|>",
"<|extra_185|>",
"<|extra_186|>",
"<|extra_187|>",
"<|extra_188|>",
"<|extra_189|>",
"<|extra_190|>",
"<|extra_191|>",
"<|extra_192|>",
"<|extra_193|>",
"<|extra_194|>",
"<|extra_195|>",
"<|extra_196|>",
"<|extra_197|>",
"<|extra_198|>",
"<|extra_199|>",
"<|extra_200|>",
"<|extra_201|>",
"<|extra_202|>",
"<|extra_203|>",
"<|extra_204|>"
],
"bos_token": "<|endoftext|>",
"clean_up_tokenization_spaces": true,
"eos_token": "<|endoftext|>",
"errors": "replace",
"model_max_length": 6144,
"pad_token": "<|endoftext|>",
"padding_side": "left",
"tokenizer_class": "GPT2Tokenizer",
"unk_token": "<|endoftext|>"
}