chandrakanthlns's picture
Upload tokenizer
e166bb0 verified
{
"&": 1,
"(": 2,
")": 3,
"[PAD]": 100,
"[UNK]": 99,
"\\": 4,
"_": 5,
"`": 6,
"a": 7,
"b": 8,
"c": 9,
"d": 10,
"e": 11,
"f": 12,
"g": 13,
"h": 14,
"i": 15,
"j": 16,
"k": 17,
"l": 18,
"m": 19,
"n": 20,
"o": 21,
"p": 22,
"q": 23,
"r": 24,
"s": 25,
"t": 26,
"u": 27,
"v": 28,
"w": 29,
"x": 30,
"y": 31,
"z": 32,
"|": 0,
"·": 33,
"½": 34,
"¾": 35,
"á": 36,
"é": 37,
"ô": 38,
"š": 39,
"ஃ": 40,
"அ": 41,
"ஆ": 42,
"இ": 43,
"ஈ": 44,
"உ": 45,
"ஊ": 46,
"எ": 47,
"ஏ": 48,
"ஐ": 49,
"ஒ": 50,
"ஓ": 51,
"ஔ": 52,
"க": 53,
"ங": 54,
"ச": 55,
"ஜ": 56,
"ஞ": 57,
"ட": 58,
"ண": 59,
"த": 60,
"ந": 61,
"ன": 62,
"ப": 63,
"ம": 64,
"ய": 65,
"ர": 66,
"ற": 67,
"ல": 68,
"ள": 69,
"ழ": 70,
"வ": 71,
"ஷ": 72,
"ஸ": 73,
"ஹ": 74,
"ா": 75,
"ி": 76,
"ீ": 77,
"ு": 78,
"ூ": 79,
"ெ": 80,
"ே": 81,
"ை": 82,
"ொ": 83,
"ோ": 84,
"ௌ": 85,
"்": 86,
"ௗ": 87,
"ഥ": 88,
"–": 89,
"—": 90,
"’": 91,
"‚": 92,
"•": 93,
"…": 94,
"′": 95,
"″": 96,
"●": 97,
"◯": 98
}