Update tokenizer.py
Browse files- tokenizer.py +1 -1
tokenizer.py
CHANGED
|
@@ -73,7 +73,7 @@ class HFAutoTokenizer:
|
|
| 73 |
return 32000
|
| 74 |
|
| 75 |
|
| 76 |
-
class ByteTokenizer:
|
| 77 |
"""UTF-8 Encoder."""
|
| 78 |
|
| 79 |
def __init__(self):
|
|
|
|
| 73 |
return 32000
|
| 74 |
|
| 75 |
|
| 76 |
+
class ByteTokenizer(PreTrainedTokenizer):
|
| 77 |
"""UTF-8 Encoder."""
|
| 78 |
|
| 79 |
def __init__(self):
|