revert tokenizer update
#9
by
Leyo
- opened
- tokenizer.json +2 -8
tokenizer.json
CHANGED
|
@@ -45,7 +45,7 @@
|
|
| 45 |
"single_word": false,
|
| 46 |
"lstrip": false,
|
| 47 |
"rstrip": false,
|
| 48 |
-
"normalized":
|
| 49 |
"special": true
|
| 50 |
},
|
| 51 |
{
|
|
@@ -71,12 +71,6 @@
|
|
| 71 |
"String": " "
|
| 72 |
},
|
| 73 |
"content": "▁"
|
| 74 |
-
},
|
| 75 |
-
{
|
| 76 |
-
"type":"Replace",
|
| 77 |
-
"pattern":{
|
| 78 |
-
"Regex":"!\\[\\]([^\\]]*)"},
|
| 79 |
-
"content":"<image>"
|
| 80 |
}
|
| 81 |
]
|
| 82 |
},
|
|
@@ -93421,4 +93415,4 @@
|
|
| 93421 |
"▁ ▁▁▁▁▁▁▁▁▁▁▁▁▁▁"
|
| 93422 |
]
|
| 93423 |
}
|
| 93424 |
-
}
|
|
|
|
| 45 |
"single_word": false,
|
| 46 |
"lstrip": false,
|
| 47 |
"rstrip": false,
|
| 48 |
+
"normalized": false,
|
| 49 |
"special": true
|
| 50 |
},
|
| 51 |
{
|
|
|
|
| 71 |
"String": " "
|
| 72 |
},
|
| 73 |
"content": "▁"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 74 |
}
|
| 75 |
]
|
| 76 |
},
|
|
|
|
| 93415 |
"▁ ▁▁▁▁▁▁▁▁▁▁▁▁▁▁"
|
| 93416 |
]
|
| 93417 |
}
|
| 93418 |
+
}
|