Reihaneh's picture
Upload tokenizer
9b966f6 verified
{
"&": 43,
"'": 23,
"(": 76,
")": 69,
"-": 46,
"D": 63,
"E": 49,
"F": 55,
"L": 41,
"N": 84,
"Y": 80,
"[": 1,
"[DE]": 90,
"[FY-NL]": 92,
"[NL]": 91,
"[PAD]": 89,
"[UNK]": 88,
"]": 14,
"a": 27,
"b": 33,
"c": 72,
"d": 70,
"e": 20,
"f": 81,
"g": 44,
"h": 38,
"i": 67,
"j": 28,
"k": 64,
"l": 83,
"m": 30,
"n": 12,
"o": 48,
"p": 16,
"q": 7,
"r": 79,
"s": 42,
"t": 62,
"u": 87,
"v": 34,
"w": 52,
"x": 78,
"y": 5,
"z": 56,
"|": 21,
"«": 50,
"´": 11,
"»": 53,
"ß": 22,
"à": 31,
"á": 66,
"â": 61,
"ä": 9,
"å": 59,
"è": 45,
"é": 71,
"ê": 6,
"ë": 25,
"í": 75,
"ï": 10,
"ñ": 4,
"ò": 65,
"ó": 40,
"ô": 39,
"ö": 17,
"ú": 36,
"û": 85,
"ü": 82,
"ý": 26,
"ā": 29,
"ă": 58,
"đ": 60,
"ı": 18,
"ň": 13,
"ő": 74,
"ş": 2,
"š": 19,
"ž": 57,
"ș": 0,
"ț": 51,
"ʿ": 73,
"ả": 24,
"ọ": 54,
"ứ": 8,
"–": 32,
"’": 77,
"“": 35,
"”": 47,
"„": 86,
"…": 15,
"→": 3,
"乡": 37,
"幺": 68
}