Fix tokenizer reloading (#42)
Browse files- fix kwarg passing (7aff27965f262d05ee7ab11e7f391ff9347e5312)
Co-authored-by: Kyle Sayers <[email protected]>
- tokenization_chatglm.py +2 -0
tokenization_chatglm.py
CHANGED
@@ -54,6 +54,8 @@ class ChatGLM4Tokenizer(PreTrainedTokenizer):
|
|
54 |
super().__init__(
|
55 |
padding_side=padding_side,
|
56 |
clean_up_tokenization_spaces=clean_up_tokenization_spaces,
|
|
|
|
|
57 |
**kwargs
|
58 |
)
|
59 |
|
|
|
54 |
super().__init__(
|
55 |
padding_side=padding_side,
|
56 |
clean_up_tokenization_spaces=clean_up_tokenization_spaces,
|
57 |
+
encode_special_tokens=encode_special_tokens,
|
58 |
+
image_size=image_size,
|
59 |
**kwargs
|
60 |
)
|
61 |
|