revert tokenizer update (#9)
Browse files- Revert "Upload tokenizer.json (#8)" (ddb9e0429104eb0d8f69110e88c8b036eeaa3f7b)
- tokenizer.json +2 -8
tokenizer.json
CHANGED
@@ -45,7 +45,7 @@
|
|
45 |
"single_word": false,
|
46 |
"lstrip": false,
|
47 |
"rstrip": false,
|
48 |
-
"normalized":
|
49 |
"special": true
|
50 |
},
|
51 |
{
|
@@ -71,12 +71,6 @@
|
|
71 |
"String": " "
|
72 |
},
|
73 |
"content": "β"
|
74 |
-
},
|
75 |
-
{
|
76 |
-
"type":"Replace",
|
77 |
-
"pattern":{
|
78 |
-
"Regex":"!\\[\\]([^\\]]*)"},
|
79 |
-
"content":"<image>"
|
80 |
}
|
81 |
]
|
82 |
},
|
@@ -93421,4 +93415,4 @@
|
|
93421 |
"β ββββββββββββββ"
|
93422 |
]
|
93423 |
}
|
93424 |
-
}
|
|
|
45 |
"single_word": false,
|
46 |
"lstrip": false,
|
47 |
"rstrip": false,
|
48 |
+
"normalized": false,
|
49 |
"special": true
|
50 |
},
|
51 |
{
|
|
|
71 |
"String": " "
|
72 |
},
|
73 |
"content": "β"
|
|
|
|
|
|
|
|
|
|
|
|
|
74 |
}
|
75 |
]
|
76 |
},
|
|
|
93415 |
"β ββββββββββββββ"
|
93416 |
]
|
93417 |
}
|
93418 |
+
}
|