Spaces:
Runtime error
Runtime error
Add cohere multilingual tokenizer
Browse files- app.py +1 -0
- dataset.csv +0 -0
app.py
CHANGED
|
@@ -40,6 +40,7 @@ tokenizer_names_to_test = [
|
|
| 40 |
"facebook/mbart-large-50",
|
| 41 |
"EleutherAI/gpt-neox-20b",
|
| 42 |
"simbolo-ai/multilingual-partial-syllable-tokenizer",
|
|
|
|
| 43 |
]
|
| 44 |
|
| 45 |
with st.sidebar:
|
|
|
|
| 40 |
"facebook/mbart-large-50",
|
| 41 |
"EleutherAI/gpt-neox-20b",
|
| 42 |
"simbolo-ai/multilingual-partial-syllable-tokenizer",
|
| 43 |
+
"Cohere/multilingual-22-12",
|
| 44 |
]
|
| 45 |
|
| 46 |
with st.sidebar:
|
dataset.csv
CHANGED
|
The diff for this file is too large to render.
See raw diff
|
|
|