{ "version": "1.0", "truncation": { "direction": "Right", "max_length": 64, "strategy": "LongestFirst", "stride": 0 }, "padding": { "strategy": "BatchLongest", "direction": "Right", "pad_to_multiple_of": null, "pad_id": 0, "pad_type_id": 0, "pad_token": "<|endoftext|>" }, "added_tokens": [ { "id": 0, "content": "<|endoftext|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 1, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 2, "content": "w", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 3, "content": "ɛ", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 4, "content": "l", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 5, "content": " ", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 6, "content": "ɪ", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 7, "content": "t", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 8, "content": "s", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 9, "content": "d", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 10, "content": "ʒ", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 11, "content": "ʌ", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 12, "content": "ð", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 13, "content": "æ", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 14, "content": "j", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 15, "content": "u", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 16, "content": "n", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 17, "content": "o", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 18, "content": "ʊ", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 19, "content": "p", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 20, "content": "a", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 21, "content": "ɔ", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 22, "content": "ɹ", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 23, "content": "h", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 24, "content": "z", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 25, "content": "e", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 26, "content": "ɑ", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 27, "content": "m", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 28, "content": "'", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 29, "content": "\n", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 30, "content": "b", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 31, "content": "i", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 32, "content": "ɜ", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 33, "content": "˞", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 34, "content": "k", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 35, "content": "ʃ", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 36, "content": "f", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 37, "content": "ɡ", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 38, "content": "ŋ", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 39, "content": "v", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 40, "content": "r", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 41, "content": "θ", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 42, "content": "ə", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 43, "content": "&", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 44, "content": ":", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 45, "content": "/", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 46, "content": "£", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 47, "content": "%", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 48, "content": ";", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 49, "content": "‘", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 50, "content": "—", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 51, "content": "$", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 52, "content": "^", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 53, "content": "↫", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 54, "content": "|", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 55, "content": "~", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 56, "content": "�", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 57, "content": "@", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 58, "content": "ˈ", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 59, "content": "⌈", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 60, "content": "⌋", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 61, "content": "⌉", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 62, "content": "⌊", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 63, "content": "≠", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 64, "content": "\\", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 65, "content": "→", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 66, "content": "◉", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 67, "content": "↗", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 68, "content": "⁎", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 69, "content": "∆", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 70, "content": "°", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 71, "content": "∇", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 72, "content": "∙", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 73, "content": "▔", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 74, "content": "☺", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 75, "content": "]", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 76, "content": "§", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 77, "content": "×", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 78, "content": "·", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 79, "content": "―", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 80, "content": "⁄", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 81, "content": "®", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 82, "content": "●", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 83, "content": "○", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 84, "content": "′", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 85, "content": "†", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 86, "content": "❌", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 87, "content": "✿", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 88, "content": "¿", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 89, "content": "–", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 90, "content": "‎", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 91, "content": "☛", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 92, "content": "☚", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 93, "content": "‡", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 94, "content": "⁂", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 95, "content": "❦", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 96, "content": "✙", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 97, "content": "÷", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 98, "content": "„", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 99, "content": "©", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 100, "content": "¢", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 101, "content": "★", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 102, "content": "“", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 103, "content": "♪", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 104, "content": "¡", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 105, "content": "¶", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 106, "content": "­", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 107, "content": "±", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 108, "content": "¬", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 109, "content": "¤", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 110, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 111, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 112, "content": "¦", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 113, "content": "¥", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 114, "content": "‹", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 115, "content": "ƒ", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 116, "content": "‚", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 117, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 118, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 119, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 120, "content": "Œ", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 121, "content": "Ž", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 122, "content": "•", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 123, "content": "’", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 124, "content": "œ", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 125, "content": "”", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 126, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 127, "content": "—", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 128, "content": "Š", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 129, "content": "ˆ", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 130, "content": "€", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 131, "content": "š", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 132, "content": "˜", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 133, "content": "–", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 134, "content": "‰", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 135, "content": "​", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 136, "content": "☻", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 137, "content": "♥", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 138, "content": "™", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 139, "content": "€", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 140, "content": "♫", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 141, "content": "‭", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 142, "content": "─", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 143, "content": "・", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 144, "content": "⬄", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 145, "content": "ž", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 146, "content": "•", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 147, "content": "。", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 148, "content": "₩", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 149, "content": "‐", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 150, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 151, "content": "☆", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 152, "content": "‚", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 153, "content": "✡", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 154, "content": "؟", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 155, "content": "≈", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 156, "content": "་", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 157, "content": "།", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 158, "content": "−", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 159, "content": "▼", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 160, "content": "₱", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 161, "content": "☐", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 162, "content": "✅", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 163, "content": "🆖", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 164, "content": "»", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 165, "content": "،", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 166, "content": "₹", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 167, "content": "‍", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 168, "content": "👽", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 169, "content": "↑", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 170, "content": "«", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 171, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 172, "content": "♠", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 173, "content": "☉", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 174, "content": "⁠", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 175, "content": "∗", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 176, "content": "⋅", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 177, "content": "‰", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 178, "content": "״", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 179, "content": "、", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 180, "content": "‏", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 181, "content": "∂", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 182, "content": "₤", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 183, "content": "✚", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 184, "content": "⟨", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 185, "content": "⟩", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 186, "content": "꞉", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 187, "content": "‌", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 188, "content": "≤", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 189, "content": "≥", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 190, "content": "⊙", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 191, "content": "∈", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 192, "content": "∉", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 193, "content": "₣", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 194, "content": "∞", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 195, "content": "😝", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 196, "content": "《", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 197, "content": "》", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 198, "content": "🜨", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 199, "content": "♭", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 200, "content": "☃", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 201, "content": "⿺", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 202, "content": "⿱", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 203, "content": "⿰", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 204, "content": "‬", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 205, "content": "※", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 206, "content": "👫", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 207, "content": "₡", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 208, "content": "『", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 209, "content": "』", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 210, "content": "☭", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 211, "content": "〜", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 212, "content": "।", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 213, "content": "॥", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 214, "content": "√", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 215, "content": "􏿾", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 216, "content": "☯", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 217, "content": "❤", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 218, "content": "♯", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 219, "content": "˧", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 220, "content": "˦", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 221, "content": "∝", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 222, "content": "◊", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 223, "content": "↊", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 224, "content": "↋", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 225, "content": "「", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 226, "content": "」", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 227, "content": "⬱", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 228, "content": "🏰", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 229, "content": "🏴", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 230, "content": "󠁧", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 231, "content": "󠁢", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 232, "content": "󠁷", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 233, "content": "󠁬", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 234, "content": "󠁳", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 235, "content": "󠁿", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 236, "content": "˥", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 237, "content": "˨", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 238, "content": "˩", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 239, "content": "≡", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 240, "content": "█", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 241, "content": "☣", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 242, "content": "♣", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 243, "content": "♦", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 244, "content": "⁣", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 245, "content": "𝄆", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 246, "content": "𝄇", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 247, "content": "≒", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 248, "content": "‖", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 249, "content": "♡", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 250, "content": "∴", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 251, "content": "᚛", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 252, "content": "᚜", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 253, "content": "←", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 254, "content": "😊", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 255, "content": "∼", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 256, "content": "₵", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 257, "content": "₿", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 258, "content": "♑", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 259, "content": "♮", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 260, "content": "‿", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 261, "content": "〉", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 262, "content": "۞", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 263, "content": "∘", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 264, "content": "♒", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 265, "content": "־", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false }, { "id": 266, "content": "♄", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false } ], "normalizer": null, "pre_tokenizer": null, "post_processor": null, "decoder": null, "model": { "type": "BPE", "dropout": null, "unk_token": null, "continuing_subword_prefix": null, "end_of_word_suffix": null, "fuse_unk": false, "byte_fallback": false, "ignore_merges": false, "vocab": {}, "merges": [] } }