Upload tokenizer
Browse files- tokenizer_script.py +1 -1
tokenizer_script.py
CHANGED
|
@@ -111,7 +111,7 @@ class CharacterTokenizer(PreTrainedTokenizer):
|
|
| 111 |
return "".join(tokens)
|
| 112 |
|
| 113 |
|
| 114 |
-
|
| 115 |
def from_json(cls, vocab_file, **kwargs):
|
| 116 |
with open(vocab_file, 'r', encoding='utf-8') as f:
|
| 117 |
vocab = json.load(f)
|
|
|
|
| 111 |
return "".join(tokens)
|
| 112 |
|
| 113 |
|
| 114 |
+
@classmethod
|
| 115 |
def from_json(cls, vocab_file, **kwargs):
|
| 116 |
with open(vocab_file, 'r', encoding='utf-8') as f:
|
| 117 |
vocab = json.load(f)
|