charlesxsh commited on
Commit
4af520c
·
1 Parent(s): dcf70a0
Files changed (1) hide show
  1. custom_tokenizer.py +1 -1
custom_tokenizer.py CHANGED
@@ -1,7 +1,7 @@
1
  from transformers import PreTrainedTokenizer, AddedToken
2
 
3
  class CustomTokenizer(PreTrainedTokenizer):
4
- def __init__(self, vocab_file, **kwargs):
5
  super().__init__(**kwargs)
6
  print("Initializing CustomTokenizer")
7
 
 
1
  from transformers import PreTrainedTokenizer, AddedToken
2
 
3
  class CustomTokenizer(PreTrainedTokenizer):
4
+ def __init__(self, **kwargs):
5
  super().__init__(**kwargs)
6
  print("Initializing CustomTokenizer")
7