manaestras commited on
Commit
cbca738
·
verified ·
1 Parent(s): 9e50fe9

Upload tokenization_hy.py with huggingface_hub

Browse files
Files changed (1) hide show
  1. tokenization_hy.py +1 -1
tokenization_hy.py CHANGED
@@ -287,7 +287,7 @@ class HYTokenizer(PreTrainedTokenizer):
287
 
288
  # tests
289
  if __name__ == "__main__":
290
- tokenizer = HYTokenizer.from_pretrained('/hy')
291
  text = '你好,世界'
292
  tokens = tokenizer.tokenize(text)
293
  print(tokens)
 
287
 
288
  # tests
289
  if __name__ == "__main__":
290
+ tokenizer = HYTokenizer.from_pretrained('./other_tokenizer_vocab/hy')
291
  text = '你好,世界'
292
  tokens = tokenizer.tokenize(text)
293
  print(tokens)