gpt2_tiny_zh-hk-wiki / added_tokens.json
jed351's picture
Upload tokenizer
cc50120
raw
history blame
2.02 kB
{
"㖭": 21128,
"㨂": 21129,
"㩒": 21130,
"䏙": 21131,
"䚗": 21132,
"䟕": 21133,
"乸": 21134,
"仼": 21135,
"冚": 21136,
"冧": 21137,
"厠": 21138,
"厹": 21139,
"吔": 21140,
"呔": 21141,
"咇": 21142,
"唒": 21143,
"唓": 21144,
"唞": 21145,
"啩": 21146,
"喼": 21147,
"嗄": 21148,
"嗌": 21149,
"嗮": 21150,
"嗱": 21151,
"嘥": 21152,
"噃": 21153,
"噉": 21154,
"噏": 21155,
"噚": 21156,
"嚹": 21157,
"嚿": 21158,
"囇": 21159,
"囯": 21160,
"妺": 21161,
"嬲": 21162,
"孭": 21163,
"屙": 21164,
"廸": 21165,
"彊": 21166,
"戇": 21167,
"戥": 21168,
"扚": 21169,
"扠": 21170,
"扺": 21171,
"扻": 21172,
"抆": 21173,
"抌": 21174,
"抺": 21175,
"捽": 21176,
"掕": 21177,
"掟": 21178,
"掹": 21179,
"揦": 21180,
"揼": 21181,
"揾": 21182,
"搣": 21183,
"搥": 21184,
"搲": 21185,
"摙": 21186,
"摷": 21187,
"撘": 21188,
"撳": 21189,
"攋": 21190,
"攰": 21191,
"曱": 21192,
"栢": 21193,
"梘": 21194,
"櫈": 21195,
"欅": 21196,
"殻": 21197,
"炆": 21198,
"烚": 21199,
"燶": 21200,
"猁": 21201,
"猢": 21202,
"獌": 21203,
"甴": 21204,
"痾": 21205,
"瘜": 21206,
"癈": 21207,
"癪": 21208,
"砵": 21209,
"祼": 21210,
"罅": 21211,
"罉": 21212,
"脷": 21213,
"蚑": 21214,
"蝻": 21215,
"袓": 21216,
"裇": 21217,
"褸": 21218,
"趷": 21219,
"跣": 21220,
"踎": 21221,
"踭": 21222,
"踼": 21223,
"躝": 21224,
"軚": 21225,
"鋭": 21226,
"錡": 21227,
"鍠": 21228,
"鎅": 21229,
"鏝": 21230,
"閂": 21231,
"閪": 21232,
"餸": 21233,
"騮": 21234,
"骹": 21235,
"髀": 21236,
"鰂": 21237,
"麖": 21238,
"黐": 21239,
"𠝹": 21240,
"𠵱": 21241,
"𠹌": 21242,
"𠺌": 21243,
"𠺘": 21244,
"𠻹": 21245,
"𠿪": 21246,
"𡃁": 21247,
"𢃇": 21248,
"𥄫": 21249,
"𨋢": 21250,
"𨳒": 21251,
"𪘲": 21252
}