Update tokenization_xgen.py
Browse files- tokenization_xgen.py +5 -1
tokenization_xgen.py
CHANGED
@@ -227,4 +227,8 @@ class XgenTokenizer(PreTrainedTokenizer):
|
|
227 |
if token_ids_1 is not None:
|
228 |
output += [1] * len(token_ids_1 + eos_token_id)
|
229 |
|
230 |
-
return output
|
|
|
|
|
|
|
|
|
|
227 |
if token_ids_1 is not None:
|
228 |
output += [1] * len(token_ids_1 + eos_token_id)
|
229 |
|
230 |
+
return output
|
231 |
+
|
232 |
+
# has no vocab file
|
233 |
+
def save_vocabulary(self, save_directory: str, filename_prefix: Optional[str] = None):
|
234 |
+
return ()
|