NX2411's picture
add tokenizer
9c2fe18
{
"1": 82,
"2": 94,
"3": 12,
"4": 97,
"5": 81,
"7": 48,
"8": 68,
"A": 77,
"B": 79,
"C": 98,
"D": 100,
"E": 62,
"F": 26,
"G": 16,
"H": 23,
"I": 58,
"J": 91,
"K": 20,
"L": 88,
"M": 102,
"N": 86,
"O": 37,
"P": 42,
"Q": 80,
"R": 15,
"S": 70,
"T": 17,
"U": 11,
"V": 108,
"W": 29,
"X": 107,
"Y": 14,
"Z": 9,
"[PAD]": 111,
"[UNK]": 110,
"a": 78,
"b": 2,
"c": 67,
"d": 59,
"e": 69,
"f": 21,
"g": 53,
"h": 33,
"i": 89,
"j": 47,
"k": 71,
"l": 57,
"m": 13,
"n": 56,
"o": 34,
"p": 61,
"q": 19,
"r": 36,
"s": 27,
"t": 64,
"u": 93,
"v": 39,
"w": 25,
"x": 32,
"y": 38,
"z": 40,
"|": 24,
"γ„±": 43,
"γ„²": 46,
"γ„³": 22,
"γ„΄": 103,
"γ„΅": 109,
"γ„Ά": 74,
"γ„·": 41,
"γ„Έ": 76,
"γ„Ή": 83,
"γ„Ί": 66,
"γ„»": 51,
"γ„Ό": 4,
"γ„Ύ": 105,
"γ„Ώ": 31,
"γ…€": 87,
"ㅁ": 52,
"γ…‚": 55,
"γ…ƒ": 35,
"γ…„": 85,
"γ……": 92,
"γ…†": 50,
"γ…‡": 7,
"γ…ˆ": 90,
"γ…‰": 84,
"γ…Š": 8,
"γ…‹": 72,
"γ…Œ": 96,
"ㅍ": 60,
"γ…Ž": 44,
"ㅏ": 95,
"ㅐ": 54,
"γ…‘": 0,
"γ…’": 101,
"γ…“": 49,
"γ…”": 45,
"γ…•": 75,
"γ…–": 106,
"γ…—": 63,
"γ…˜": 10,
"γ…™": 104,
"γ…š": 6,
"γ…›": 3,
"γ…œ": 1,
"ㅝ": 99,
"γ…ž": 18,
"γ…Ÿ": 30,
"γ… ": 28,
"γ…‘": 5,
"γ…’": 65,
"γ…£": 73
}