Mia Xia
commited on
Commit
•
bbd743f
1
Parent(s):
b25e4ca
Upload tokenizer.json
Browse files- tokenizer.json +6 -6
tokenizer.json
CHANGED
@@ -2116,17 +2116,17 @@
|
|
2116 |
},
|
2117 |
"content": " "
|
2118 |
},
|
2119 |
-
{
|
2120 |
-
"type": "Strip",
|
2121 |
-
"content": " ",
|
2122 |
-
"start": 1,
|
2123 |
-
"stop": 0
|
2124 |
-
},
|
2125 |
{
|
2126 |
"type": "ByteFallback"
|
2127 |
},
|
2128 |
{
|
2129 |
"type": "Fuse"
|
|
|
|
|
|
|
|
|
|
|
|
|
2130 |
}
|
2131 |
]
|
2132 |
},
|
|
|
2116 |
},
|
2117 |
"content": " "
|
2118 |
},
|
|
|
|
|
|
|
|
|
|
|
|
|
2119 |
{
|
2120 |
"type": "ByteFallback"
|
2121 |
},
|
2122 |
{
|
2123 |
"type": "Fuse"
|
2124 |
+
},
|
2125 |
+
{
|
2126 |
+
"type": "Strip",
|
2127 |
+
"content": " ",
|
2128 |
+
"start": 1,
|
2129 |
+
"stop": 0
|
2130 |
}
|
2131 |
]
|
2132 |
},
|