Upload tokenization_hy.py with huggingface_hub
Browse files- tokenization_hy.py +1 -1
tokenization_hy.py
CHANGED
@@ -287,7 +287,7 @@ class HYTokenizer(PreTrainedTokenizer):
|
|
287 |
|
288 |
# tests
|
289 |
if __name__ == "__main__":
|
290 |
-
tokenizer = HYTokenizer.from_pretrained('/hy')
|
291 |
text = '你好,世界'
|
292 |
tokens = tokenizer.tokenize(text)
|
293 |
print(tokens)
|
|
|
287 |
|
288 |
# tests
|
289 |
if __name__ == "__main__":
|
290 |
+
tokenizer = HYTokenizer.from_pretrained('./other_tokenizer_vocab/hy')
|
291 |
text = '你好,世界'
|
292 |
tokens = tokenizer.tokenize(text)
|
293 |
print(tokens)
|